{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-02T00:32:06.965218Z",
     "iopub.status.busy": "2021-07-02T00:32:06.964650Z",
     "iopub.status.idle": "2021-07-02T00:32:06.970275Z",
     "shell.execute_reply": "2021-07-02T00:32:06.969201Z",
     "shell.execute_reply.started": "2021-07-02T00:32:06.965166Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# https://github.com/skdjfla/toutiao-text-classfication-dataset/raw/master/toutiao_cat_data.txt.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-24T06:04:51.923802Z",
     "iopub.status.busy": "2021-04-24T06:04:51.923281Z",
     "iopub.status.idle": "2021-04-24T06:04:52.468096Z",
     "shell.execute_reply": "2021-04-24T06:04:52.466703Z",
     "shell.execute_reply.started": "2021-04-24T06:04:51.923756Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Archive:  toutiao_cat_data.txt.zip\n",
      "  inflating: toutiao_cat_data.txt    \n"
     ]
    }
   ],
   "source": [
    "!unzip toutiao_cat_data.txt.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-24T06:05:08.025701Z",
     "iopub.status.busy": "2021-04-24T06:05:08.025175Z",
     "iopub.status.idle": "2021-04-24T06:05:08.146121Z",
     "shell.execute_reply": "2021-04-24T06:05:08.145029Z",
     "shell.execute_reply.started": "2021-04-24T06:05:08.025656Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6551700932705387022_!_101_!_news_culture_!_京城最值得你来场文化之旅的博物馆_!_保利集团,马未都,中国科学技术馆,博物馆,新中国\n",
      "6552368441838272771_!_101_!_news_culture_!_发酵床的垫料种类有哪些？哪种更好？_!_\n",
      "6552407965343678723_!_101_!_news_culture_!_上联：黄山黄河黄皮肤黄土高原。怎么对下联？_!_\n",
      "6552332417753940238_!_101_!_news_culture_!_林徽因什么理由拒绝了徐志摩而选择梁思成为终身伴侣？_!_\n",
      "6552475601595269390_!_101_!_news_culture_!_黄杨木是什么树？_!_\n",
      "6552387648126714125_!_101_!_news_culture_!_上联：草根登上星光道，怎么对下联？_!_\n",
      "6552271725814350087_!_101_!_news_culture_!_什么是超写实绘画？_!_\n",
      "6552452982015787268_!_101_!_news_culture_!_松涛听雨莺婉转，下联？_!_\n",
      "6552400379030536455_!_101_!_news_culture_!_上联：老子骑牛读书，下联怎么对？_!_\n",
      "6552339283632455939_!_101_!_news_culture_!_上联：山水醉人何须酒。如何对下联？_!_\n"
     ]
    }
   ],
   "source": [
    "!head toutiao_cat_data.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-08-09T10:34:03.467099Z",
     "iopub.status.busy": "2021-08-09T10:34:03.466498Z",
     "iopub.status.idle": "2021-08-09T10:34:04.539061Z",
     "shell.execute_reply": "2021-08-09T10:34:04.538035Z",
     "shell.execute_reply.started": "2021-08-09T10:34:03.467048Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# pandas 数据集读取，dataframe形式的\n",
    "import pandas as pd\n",
    "# 文件读取\n",
    "import codecs\n",
    "\n",
    "# 读取文本\n",
    "\n",
    "# 标签\n",
    "news_label = [int(x.split('_!_')[1])-100 \n",
    "                  for x in codecs.open('toutiao_cat_data.txt')]\n",
    "\n",
    "# 文本\n",
    "news_text = [x.strip().split('_!_')[-1] if x.strip()[-3:] != '_!_' else x.strip().split('_!_')[-2]\n",
    "                 for x in codecs.open('toutiao_cat_data.txt')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-11T09:35:08.358976Z",
     "start_time": "2021-03-11T09:35:08.347065Z"
    },
    "execution": {
     "iopub.execute_input": "2021-08-09T10:34:07.737158Z",
     "iopub.status.busy": "2021-08-09T10:34:07.736631Z",
     "iopub.status.idle": "2021-08-09T10:34:08.674703Z",
     "shell.execute_reply": "2021-08-09T10:34:08.674100Z",
     "shell.execute_reply.started": "2021-08-09T10:34:07.737123Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "from sklearn.model_selection import train_test_split\n",
    "from torch.utils.data import Dataset, DataLoader, TensorDataset\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import random\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-11T09:35:09.263768Z",
     "start_time": "2021-03-11T09:35:09.227357Z"
    },
    "execution": {
     "iopub.execute_input": "2021-08-09T10:34:08.804028Z",
     "iopub.status.busy": "2021-08-09T10:34:08.803494Z",
     "iopub.status.idle": "2021-08-09T10:34:08.950219Z",
     "shell.execute_reply": "2021-08-09T10:34:08.949655Z",
     "shell.execute_reply.started": "2021-08-09T10:34:08.803981Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 划分为训练集和验证集\n",
    "# stratify 按照标签进行采样，训练集和验证部分同分布\n",
    "x_train, x_test, train_label, test_label =  train_test_split(news_text[:50000], \n",
    "                                                             news_label[:50000], \n",
    "                                                             test_size=0.2, \n",
    "                                                             stratify=news_label[:50000])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-08-09T10:34:10.614817Z",
     "iopub.status.busy": "2021-08-09T10:34:10.614210Z",
     "iopub.status.idle": "2021-08-09T10:34:10.619388Z",
     "shell.execute_reply": "2021-08-09T10:34:10.618087Z",
     "shell.execute_reply.started": "2021-08-09T10:34:10.614767Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# input_ids：字的编码\n",
    "# token_type_ids：标识是第一个句子还是第二个句子\n",
    "# attention_mask：标识是不是填充"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-11T09:35:19.935035Z",
     "start_time": "2021-03-11T09:35:09.908638Z"
    },
    "execution": {
     "iopub.execute_input": "2021-08-09T10:34:10.951013Z",
     "iopub.status.busy": "2021-08-09T10:34:10.950447Z",
     "iopub.status.idle": "2021-08-09T10:34:24.367488Z",
     "shell.execute_reply": "2021-08-09T10:34:24.366777Z",
     "shell.execute_reply.started": "2021-08-09T10:34:10.950963Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# pip install transformers\n",
    "# transformers bert相关的模型使用和加载\n",
    "from transformers import BertTokenizer\n",
    "# 分词器，词典\n",
    "\n",
    "tokenizer = BertTokenizer.from_pretrained('bert-base-chinese')\n",
    "train_encoding = tokenizer(x_train, truncation=True, padding=True, max_length=64)\n",
    "test_encoding = tokenizer(x_test, truncation=True, padding=True, max_length=64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-08-09T10:34:39.590239Z",
     "iopub.status.busy": "2021-08-09T10:34:39.589656Z",
     "iopub.status.idle": "2021-08-09T10:34:39.599674Z",
     "shell.execute_reply": "2021-08-09T10:34:39.599053Z",
     "shell.execute_reply.started": "2021-08-09T10:34:39.590190Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'input_ids': [101, 2769, 812, 3221, 1962, 2110, 4495, 102, 1962, 2110, 4495, 3221, 2769, 812, 102], 'token_type_ids': [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer('我们是好学生', '好学生是我们',  padding=True, max_length=120)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-11T09:35:43.578135Z",
     "start_time": "2021-03-11T09:35:43.571452Z"
    },
    "execution": {
     "iopub.execute_input": "2021-08-09T10:34:39.924868Z",
     "iopub.status.busy": "2021-08-09T10:34:39.924320Z",
     "iopub.status.idle": "2021-08-09T10:34:39.934628Z",
     "shell.execute_reply": "2021-08-09T10:34:39.934063Z",
     "shell.execute_reply.started": "2021-08-09T10:34:39.924819Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 数据集读取\n",
    "class NewsDataset(Dataset):\n",
    "    def __init__(self, encodings, labels):\n",
    "        self.encodings = encodings\n",
    "        self.labels = labels\n",
    "    \n",
    "    # 读取单个样本\n",
    "    def __getitem__(self, idx):\n",
    "        item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
    "        item['labels'] = torch.tensor(int(self.labels[idx]))\n",
    "        return item\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.labels)\n",
    "\n",
    "train_dataset = NewsDataset(train_encoding, train_label)\n",
    "test_dataset = NewsDataset(test_encoding, test_label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-08-09T10:34:40.907034Z",
     "iopub.status.busy": "2021-08-09T10:34:40.906436Z",
     "iopub.status.idle": "2021-08-09T10:34:40.915213Z",
     "shell.execute_reply": "2021-08-09T10:34:40.914169Z",
     "shell.execute_reply.started": "2021-08-09T10:34:40.906985Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'input_ids': tensor([ 101, 2341, 1046, 1213,  117, 1788, 1046, 2797,  117, 2773, 3159, 3322,\n",
       "          117, 1957, 7607, 6121, 1447,  117, 2773, 3159, 3322, 7607, 6121, 1447,\n",
       "          102,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0]),\n",
       " 'token_type_ids': tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]),\n",
       " 'attention_mask': tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "         1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]),\n",
       " 'labels': tensor(10)}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_dataset[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-11T09:35:44.110121Z",
     "start_time": "2021-03-11T09:35:44.104871Z"
    },
    "execution": {
     "iopub.execute_input": "2021-08-09T10:34:41.802112Z",
     "iopub.status.busy": "2021-08-09T10:34:41.801669Z",
     "iopub.status.idle": "2021-08-09T10:34:41.807352Z",
     "shell.execute_reply": "2021-08-09T10:34:41.806316Z",
     "shell.execute_reply.started": "2021-08-09T10:34:41.802078Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 精度计算\n",
    "def flat_accuracy(preds, labels):\n",
    "    pred_flat = np.argmax(preds, axis=1).flatten()\n",
    "    labels_flat = labels.flatten()\n",
    "    return np.sum(pred_flat == labels_flat) / len(labels_flat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-11T09:58:49.027161Z",
     "start_time": "2021-03-11T09:58:45.317009Z"
    },
    "execution": {
     "iopub.execute_input": "2021-08-09T10:34:42.856891Z",
     "iopub.status.busy": "2021-08-09T10:34:42.856311Z",
     "iopub.status.idle": "2021-08-09T10:35:03.639620Z",
     "shell.execute_reply": "2021-08-09T10:35:03.638888Z",
     "shell.execute_reply.started": "2021-08-09T10:34:42.856843Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at bert-base-chinese were not used when initializing BertForSequenceClassification: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias']\n",
      "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-chinese and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "from transformers import BertForSequenceClassification, AdamW, get_linear_schedule_with_warmup\n",
    "model = BertForSequenceClassification.from_pretrained('bert-base-chinese', num_labels=17)\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "model.to(device)\n",
    "\n",
    "# 单个读取到批量读取\n",
    "train_loader = DataLoader(train_dataset, batch_size=16, shuffle=True)\n",
    "test_dataloader = DataLoader(test_dataset, batch_size=16, shuffle=True)\n",
    "\n",
    "# 优化方法\n",
    "optim = AdamW(model.parameters(), lr=2e-5)\n",
    "total_steps = len(train_loader) * 1\n",
    "scheduler = get_linear_schedule_with_warmup(optim, \n",
    "                                            num_warmup_steps = 0, # Default value in run_glue.py\n",
    "                                            num_training_steps = total_steps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-11T09:44:22.077501Z",
     "start_time": "2021-03-11T09:39:16.473609Z"
    },
    "execution": {
     "iopub.execute_input": "2021-04-24T06:32:30.552523Z",
     "iopub.status.busy": "2021-04-24T06:32:30.551970Z",
     "iopub.status.idle": "2021-04-24T06:50:38.809648Z",
     "shell.execute_reply": "2021-04-24T06:50:38.807666Z",
     "shell.execute_reply.started": "2021-04-24T06:32:30.552474Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------Epoch: 0 ----------------\n",
      "epoth: 0, iter_num: 100, loss: 1.1153, 4.00%\n",
      "epoth: 0, iter_num: 200, loss: 0.6116, 8.00%\n",
      "epoth: 0, iter_num: 300, loss: 1.2130, 12.00%\n",
      "epoth: 0, iter_num: 400, loss: 0.2496, 16.00%\n",
      "epoth: 0, iter_num: 500, loss: 0.4285, 20.00%\n",
      "epoth: 0, iter_num: 600, loss: 0.3757, 24.00%\n",
      "epoth: 0, iter_num: 700, loss: 0.6119, 28.00%\n",
      "epoth: 0, iter_num: 800, loss: 0.3269, 32.00%\n",
      "epoth: 0, iter_num: 900, loss: 0.5932, 36.00%\n",
      "epoth: 0, iter_num: 1000, loss: 0.2557, 40.00%\n",
      "epoth: 0, iter_num: 1100, loss: 0.3224, 44.00%\n",
      "epoth: 0, iter_num: 1200, loss: 0.2232, 48.00%\n",
      "epoth: 0, iter_num: 1300, loss: 0.7005, 52.00%\n",
      "epoth: 0, iter_num: 1400, loss: 0.4618, 56.00%\n",
      "epoth: 0, iter_num: 1500, loss: 0.3498, 60.00%\n",
      "epoth: 0, iter_num: 1600, loss: 1.2221, 64.00%\n",
      "epoth: 0, iter_num: 1700, loss: 0.3283, 68.00%\n",
      "epoth: 0, iter_num: 1800, loss: 0.1679, 72.00%\n",
      "epoth: 0, iter_num: 1900, loss: 0.3219, 76.00%\n",
      "epoth: 0, iter_num: 2000, loss: 0.6863, 80.00%\n",
      "epoth: 0, iter_num: 2100, loss: 0.3656, 84.00%\n",
      "epoth: 0, iter_num: 2200, loss: 0.1841, 88.00%\n",
      "epoth: 0, iter_num: 2300, loss: 0.2048, 92.00%\n",
      "epoth: 0, iter_num: 2400, loss: 0.6750, 96.00%\n",
      "epoth: 0, iter_num: 2500, loss: 0.7519, 100.00%\n",
      "Epoch: 0, Average training loss: 0.6181\n",
      "Accuracy: 0.8747\n",
      "Average testing loss: 0.4602\n",
      "-------------------------------\n",
      "------------Epoch: 1 ----------------\n",
      "epoth: 1, iter_num: 100, loss: 0.4814, 4.00%\n",
      "epoth: 1, iter_num: 200, loss: 0.1946, 8.00%\n",
      "epoth: 1, iter_num: 300, loss: 0.1796, 12.00%\n",
      "epoth: 1, iter_num: 400, loss: 0.0931, 16.00%\n",
      "epoth: 1, iter_num: 500, loss: 0.5104, 20.00%\n",
      "epoth: 1, iter_num: 600, loss: 0.4047, 24.00%\n",
      "epoth: 1, iter_num: 700, loss: 0.0419, 28.00%\n",
      "epoth: 1, iter_num: 800, loss: 0.3464, 32.00%\n",
      "epoth: 1, iter_num: 900, loss: 0.2367, 36.00%\n",
      "epoth: 1, iter_num: 1000, loss: 0.6062, 40.00%\n",
      "epoth: 1, iter_num: 1100, loss: 0.5128, 44.00%\n",
      "epoth: 1, iter_num: 1200, loss: 0.4490, 48.00%\n",
      "epoth: 1, iter_num: 1300, loss: 0.3533, 52.00%\n",
      "epoth: 1, iter_num: 1400, loss: 0.8225, 56.00%\n",
      "epoth: 1, iter_num: 1500, loss: 0.0551, 60.00%\n",
      "epoth: 1, iter_num: 1600, loss: 0.3125, 64.00%\n",
      "epoth: 1, iter_num: 1700, loss: 0.3223, 68.00%\n",
      "epoth: 1, iter_num: 1800, loss: 0.2906, 72.00%\n",
      "epoth: 1, iter_num: 1900, loss: 0.8114, 76.00%\n",
      "epoth: 1, iter_num: 2000, loss: 0.9646, 80.00%\n",
      "epoth: 1, iter_num: 2100, loss: 0.4762, 84.00%\n",
      "epoth: 1, iter_num: 2200, loss: 0.2437, 88.00%\n",
      "epoth: 1, iter_num: 2300, loss: 0.0382, 92.00%\n",
      "epoth: 1, iter_num: 2400, loss: 0.3108, 96.00%\n",
      "epoth: 1, iter_num: 2500, loss: 0.4420, 100.00%\n",
      "Epoch: 1, Average training loss: 0.3609\n",
      "Accuracy: 0.8747\n",
      "Average testing loss: 0.4602\n",
      "-------------------------------\n",
      "------------Epoch: 2 ----------------\n",
      "epoth: 2, iter_num: 100, loss: 0.4151, 4.00%\n",
      "epoth: 2, iter_num: 200, loss: 0.4348, 8.00%\n",
      "epoth: 2, iter_num: 300, loss: 0.1247, 12.00%\n",
      "epoth: 2, iter_num: 400, loss: 0.1380, 16.00%\n",
      "epoth: 2, iter_num: 500, loss: 0.0994, 20.00%\n",
      "epoth: 2, iter_num: 600, loss: 0.3140, 24.00%\n",
      "epoth: 2, iter_num: 700, loss: 0.0777, 28.00%\n",
      "epoth: 2, iter_num: 800, loss: 0.4238, 32.00%\n",
      "epoth: 2, iter_num: 900, loss: 0.4849, 36.00%\n",
      "epoth: 2, iter_num: 1000, loss: 0.3703, 40.00%\n",
      "epoth: 2, iter_num: 1100, loss: 0.4374, 44.00%\n",
      "epoth: 2, iter_num: 1200, loss: 0.1970, 48.00%\n",
      "epoth: 2, iter_num: 1300, loss: 0.4953, 52.00%\n",
      "epoth: 2, iter_num: 1400, loss: 1.0947, 56.00%\n",
      "epoth: 2, iter_num: 1500, loss: 0.4093, 60.00%\n",
      "epoth: 2, iter_num: 1600, loss: 0.2815, 64.00%\n",
      "epoth: 2, iter_num: 1700, loss: 0.7125, 68.00%\n",
      "epoth: 2, iter_num: 1800, loss: 0.4488, 72.00%\n",
      "epoth: 2, iter_num: 1900, loss: 0.6786, 76.00%\n",
      "epoth: 2, iter_num: 2000, loss: 0.1705, 80.00%\n",
      "epoth: 2, iter_num: 2100, loss: 0.2752, 84.00%\n",
      "epoth: 2, iter_num: 2200, loss: 1.2604, 88.00%\n",
      "epoth: 2, iter_num: 2300, loss: 0.4147, 92.00%\n",
      "epoth: 2, iter_num: 2400, loss: 0.3955, 96.00%\n",
      "epoth: 2, iter_num: 2500, loss: 0.6593, 100.00%\n",
      "Epoch: 2, Average training loss: 0.3608\n",
      "Accuracy: 0.8747\n",
      "Average testing loss: 0.4602\n",
      "-------------------------------\n",
      "------------Epoch: 3 ----------------\n",
      "epoth: 3, iter_num: 100, loss: 0.1005, 4.00%\n",
      "epoth: 3, iter_num: 200, loss: 0.2495, 8.00%\n",
      "epoth: 3, iter_num: 300, loss: 0.3578, 12.00%\n",
      "epoth: 3, iter_num: 400, loss: 0.5624, 16.00%\n",
      "epoth: 3, iter_num: 500, loss: 0.3800, 20.00%\n",
      "epoth: 3, iter_num: 600, loss: 0.1445, 24.00%\n",
      "epoth: 3, iter_num: 700, loss: 0.4523, 28.00%\n",
      "epoth: 3, iter_num: 800, loss: 0.0616, 32.00%\n",
      "epoth: 3, iter_num: 900, loss: 0.2931, 36.00%\n",
      "epoth: 3, iter_num: 1000, loss: 0.8472, 40.00%\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-8-de62f035659a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     50\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mepoch\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m4\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     51\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"------------Epoch: %d ----------------\"\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mepoch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 52\u001b[0;31m     \u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     53\u001b[0m     \u001b[0mvalidation\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-8-de62f035659a>\u001b[0m in \u001b[0;36mtrain\u001b[0;34m()\u001b[0m\n\u001b[1;32m     12\u001b[0m         \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0moutputs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     13\u001b[0m         \u001b[0mtotal_train_loss\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 14\u001b[0;31m         \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     15\u001b[0m         \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mutils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclip_grad_norm_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparameters\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1.0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     16\u001b[0m         \u001b[0moptim\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/torch/tensor.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(self, gradient, retain_graph, create_graph)\u001b[0m\n\u001b[1;32m    219\u001b[0m                 \u001b[0mretain_graph\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    220\u001b[0m                 create_graph=create_graph)\n\u001b[0;32m--> 221\u001b[0;31m         \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgradient\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    222\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    223\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mregister_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/torch/autograd/__init__.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables)\u001b[0m\n\u001b[1;32m    130\u001b[0m     Variable._execution_engine.run_backward(\n\u001b[1;32m    131\u001b[0m         \u001b[0mtensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad_tensors_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 132\u001b[0;31m         allow_unreachable=True)  # allow_unreachable flag\n\u001b[0m\u001b[1;32m    133\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    134\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# 训练函数\n",
    "def train():\n",
    "    model.train()\n",
    "    total_train_loss = 0\n",
    "    iter_num = 0\n",
    "    total_iter = len(train_loader)\n",
    "    for batch in train_loader:\n",
    "        # 正向传播\n",
    "        optim.zero_grad()\n",
    "        \n",
    "        input_ids = batch['input_ids'].to(device)\n",
    "        attention_mask = batch['attention_mask'].to(device)\n",
    "        labels = batch['labels'].to(device)\n",
    "        \n",
    "        outputs = model(input_ids, attention_mask=attention_mask, labels=labels)\n",
    "        loss = outputs[0]\n",
    "        total_train_loss += loss.item()\n",
    "        \n",
    "        # 反向梯度信息\n",
    "        loss.backward()\n",
    "        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n",
    "        \n",
    "        # 参数更新\n",
    "        optim.step()\n",
    "        scheduler.step()\n",
    "\n",
    "        iter_num += 1\n",
    "        if(iter_num % 100==0):\n",
    "            print(\"epoth: %d, iter_num: %d, loss: %.4f, %.2f%%\" % (epoch, iter_num, loss.item(), iter_num/total_iter*100))\n",
    "        \n",
    "    print(\"Epoch: %d, Average training loss: %.4f\"%(epoch, total_train_loss/len(train_loader)))\n",
    "    \n",
    "def validation():\n",
    "    model.eval()\n",
    "    total_eval_accuracy = 0\n",
    "    total_eval_loss = 0\n",
    "    for batch in test_dataloader:\n",
    "        with torch.no_grad():\n",
    "            # 正常传播\n",
    "            input_ids = batch['input_ids'].to(device)\n",
    "            attention_mask = batch['attention_mask'].to(device)\n",
    "            labels = batch['labels'].to(device)\n",
    "            outputs = model(input_ids, attention_mask=attention_mask, labels=labels)\n",
    "        \n",
    "        loss = outputs[0]\n",
    "        logits = outputs[1]\n",
    "\n",
    "        total_eval_loss += loss.item()\n",
    "        logits = logits.detach().cpu().numpy()\n",
    "        label_ids = labels.to('cpu').numpy()\n",
    "        total_eval_accuracy += flat_accuracy(logits, label_ids)\n",
    "        \n",
    "    avg_val_accuracy = total_eval_accuracy / len(test_dataloader)\n",
    "    print(\"Accuracy: %.4f\" % (avg_val_accuracy))\n",
    "    print(\"Average testing loss: %.4f\"%(total_eval_loss/len(test_dataloader)))\n",
    "    print(\"-------------------------------\")\n",
    "    \n",
    "\n",
    "for epoch in range(4):\n",
    "    print(\"------------Epoch: %d ----------------\" % epoch)\n",
    "    train()\n",
    "    validation()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-08-09T10:35:12.482535Z",
     "iopub.status.busy": "2021-08-09T10:35:12.481939Z",
     "iopub.status.idle": "2021-08-09T10:35:12.489156Z",
     "shell.execute_reply": "2021-08-09T10:35:12.488457Z",
     "shell.execute_reply.started": "2021-08-09T10:35:12.482480Z"
    }
   },
   "outputs": [],
   "source": [
    "class FGM():\n",
    "    def __init__(self, model):\n",
    "        self.model = model\n",
    "        self.backup = {}\n",
    "\n",
    "    def attack(self, epsilon=1., emb_name='emb'):\n",
    "        # emb_name这个参数要换成你模型中embedding的参数名\n",
    "        # 例如，self.emb = nn.Embedding(5000, 100)\n",
    "        for name, param in self.model.named_parameters():\n",
    "            if param.requires_grad and 'embedding' in name:\n",
    "                self.backup[name] = param.data.clone()\n",
    "                norm = torch.norm(param.grad) # 默认为2范数\n",
    "                if norm != 0:\n",
    "                    r_at = epsilon * param.grad / norm\n",
    "                    param.data.add_(r_at)\n",
    "\n",
    "    def restore(self, emb_name='emb'):\n",
    "        # emb_name这个参数要换成你模型中embedding的参数名\n",
    "        for name, param in self.model.named_parameters():\n",
    "            if param.requires_grad and 'embedding' in name: \n",
    "                assert name in self.backup\n",
    "                param.data = self.backup[name]\n",
    "        self.backup = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-08-09T10:36:50.668629Z",
     "iopub.status.busy": "2021-08-09T10:36:50.668041Z",
     "iopub.status.idle": "2021-08-09T11:13:11.564537Z",
     "shell.execute_reply": "2021-08-09T11:13:11.563967Z",
     "shell.execute_reply.started": "2021-08-09T10:36:50.668580Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------Epoch: 0 ----------------\n",
      "epoth: 0, iter_num: 10, loss: 2.8839, 0.40%\n",
      "epoth: 0, iter_num: 20, loss: 2.7609, 0.80%\n",
      "epoth: 0, iter_num: 30, loss: 2.5918, 1.20%\n",
      "epoth: 0, iter_num: 40, loss: 2.6522, 1.60%\n",
      "epoth: 0, iter_num: 50, loss: 2.7058, 2.00%\n",
      "epoth: 0, iter_num: 60, loss: 2.6441, 2.40%\n",
      "epoth: 0, iter_num: 70, loss: 2.6895, 2.80%\n",
      "epoth: 0, iter_num: 80, loss: 2.6126, 3.20%\n",
      "epoth: 0, iter_num: 90, loss: 2.7909, 3.60%\n",
      "epoth: 0, iter_num: 100, loss: 2.6045, 4.00%\n",
      "epoth: 0, iter_num: 110, loss: 2.6010, 4.40%\n",
      "epoth: 0, iter_num: 120, loss: 2.6245, 4.80%\n",
      "epoth: 0, iter_num: 130, loss: 2.5980, 5.20%\n",
      "epoth: 0, iter_num: 140, loss: 2.6549, 5.60%\n",
      "epoth: 0, iter_num: 150, loss: 2.6406, 6.00%\n",
      "epoth: 0, iter_num: 160, loss: 2.5500, 6.40%\n",
      "epoth: 0, iter_num: 170, loss: 2.6483, 6.80%\n",
      "epoth: 0, iter_num: 180, loss: 2.5613, 7.20%\n",
      "epoth: 0, iter_num: 190, loss: 2.6371, 7.60%\n",
      "epoth: 0, iter_num: 200, loss: 2.5517, 8.00%\n",
      "epoth: 0, iter_num: 210, loss: 2.4984, 8.40%\n",
      "epoth: 0, iter_num: 220, loss: 2.5130, 8.80%\n",
      "epoth: 0, iter_num: 230, loss: 2.6860, 9.20%\n",
      "epoth: 0, iter_num: 240, loss: 2.6014, 9.60%\n",
      "epoth: 0, iter_num: 250, loss: 2.6091, 10.00%\n",
      "epoth: 0, iter_num: 260, loss: 2.6149, 10.40%\n",
      "epoth: 0, iter_num: 270, loss: 2.6098, 10.80%\n",
      "epoth: 0, iter_num: 280, loss: 2.5798, 11.20%\n",
      "epoth: 0, iter_num: 290, loss: 2.7097, 11.60%\n",
      "epoth: 0, iter_num: 300, loss: 2.5436, 12.00%\n",
      "epoth: 0, iter_num: 310, loss: 2.5809, 12.40%\n",
      "epoth: 0, iter_num: 320, loss: 2.5866, 12.80%\n",
      "epoth: 0, iter_num: 330, loss: 2.6490, 13.20%\n",
      "epoth: 0, iter_num: 340, loss: 2.4599, 13.60%\n",
      "epoth: 0, iter_num: 350, loss: 2.6238, 14.00%\n",
      "epoth: 0, iter_num: 360, loss: 2.4276, 14.40%\n",
      "epoth: 0, iter_num: 370, loss: 2.5751, 14.80%\n",
      "epoth: 0, iter_num: 380, loss: 2.6072, 15.20%\n",
      "epoth: 0, iter_num: 390, loss: 2.6294, 15.60%\n",
      "epoth: 0, iter_num: 400, loss: 2.5479, 16.00%\n",
      "epoth: 0, iter_num: 410, loss: 2.5632, 16.40%\n",
      "epoth: 0, iter_num: 420, loss: 2.4474, 16.80%\n",
      "epoth: 0, iter_num: 430, loss: 2.6475, 17.20%\n",
      "epoth: 0, iter_num: 440, loss: 2.5324, 17.60%\n",
      "epoth: 0, iter_num: 450, loss: 2.4847, 18.00%\n",
      "epoth: 0, iter_num: 460, loss: 2.5948, 18.40%\n",
      "epoth: 0, iter_num: 470, loss: 2.5414, 18.80%\n",
      "epoth: 0, iter_num: 480, loss: 2.5710, 19.20%\n",
      "epoth: 0, iter_num: 490, loss: 2.5099, 19.60%\n",
      "epoth: 0, iter_num: 500, loss: 2.5902, 20.00%\n",
      "epoth: 0, iter_num: 510, loss: 2.5889, 20.40%\n",
      "epoth: 0, iter_num: 520, loss: 2.5315, 20.80%\n",
      "epoth: 0, iter_num: 530, loss: 2.5100, 21.20%\n",
      "epoth: 0, iter_num: 540, loss: 2.5320, 21.60%\n",
      "epoth: 0, iter_num: 550, loss: 2.6010, 22.00%\n",
      "epoth: 0, iter_num: 560, loss: 2.5586, 22.40%\n",
      "epoth: 0, iter_num: 570, loss: 2.5071, 22.80%\n",
      "epoth: 0, iter_num: 580, loss: 2.5947, 23.20%\n",
      "epoth: 0, iter_num: 590, loss: 2.6091, 23.60%\n",
      "epoth: 0, iter_num: 600, loss: 2.6278, 24.00%\n",
      "epoth: 0, iter_num: 610, loss: 2.4869, 24.40%\n",
      "epoth: 0, iter_num: 620, loss: 2.5338, 24.80%\n",
      "epoth: 0, iter_num: 630, loss: 2.5438, 25.20%\n",
      "epoth: 0, iter_num: 640, loss: 2.7135, 25.60%\n",
      "epoth: 0, iter_num: 650, loss: 2.4988, 26.00%\n",
      "epoth: 0, iter_num: 660, loss: 2.5380, 26.40%\n",
      "epoth: 0, iter_num: 670, loss: 2.6300, 26.80%\n",
      "epoth: 0, iter_num: 680, loss: 2.5529, 27.20%\n",
      "epoth: 0, iter_num: 690, loss: 2.6445, 27.60%\n",
      "epoth: 0, iter_num: 700, loss: 2.6359, 28.00%\n",
      "epoth: 0, iter_num: 710, loss: 2.5120, 28.40%\n",
      "epoth: 0, iter_num: 720, loss: 2.5556, 28.80%\n",
      "epoth: 0, iter_num: 730, loss: 2.4890, 29.20%\n",
      "epoth: 0, iter_num: 740, loss: 2.5582, 29.60%\n",
      "epoth: 0, iter_num: 750, loss: 2.4002, 30.00%\n",
      "epoth: 0, iter_num: 760, loss: 2.5075, 30.40%\n",
      "epoth: 0, iter_num: 770, loss: 2.6173, 30.80%\n",
      "epoth: 0, iter_num: 780, loss: 2.5109, 31.20%\n",
      "epoth: 0, iter_num: 790, loss: 2.5812, 31.60%\n",
      "epoth: 0, iter_num: 800, loss: 2.4427, 32.00%\n",
      "epoth: 0, iter_num: 810, loss: 2.4063, 32.40%\n",
      "epoth: 0, iter_num: 820, loss: 2.3589, 32.80%\n",
      "epoth: 0, iter_num: 830, loss: 2.4481, 33.20%\n",
      "epoth: 0, iter_num: 840, loss: 2.5095, 33.60%\n",
      "epoth: 0, iter_num: 850, loss: 2.4429, 34.00%\n",
      "epoth: 0, iter_num: 860, loss: 2.4872, 34.40%\n",
      "epoth: 0, iter_num: 870, loss: 2.4171, 34.80%\n",
      "epoth: 0, iter_num: 880, loss: 2.3384, 35.20%\n",
      "epoth: 0, iter_num: 890, loss: 2.4321, 35.60%\n",
      "epoth: 0, iter_num: 900, loss: 2.3816, 36.00%\n",
      "epoth: 0, iter_num: 910, loss: 2.1891, 36.40%\n",
      "epoth: 0, iter_num: 920, loss: 2.3268, 36.80%\n",
      "epoth: 0, iter_num: 930, loss: 2.3561, 37.20%\n",
      "epoth: 0, iter_num: 940, loss: 2.3059, 37.60%\n",
      "epoth: 0, iter_num: 950, loss: 2.5038, 38.00%\n",
      "epoth: 0, iter_num: 960, loss: 2.2944, 38.40%\n",
      "epoth: 0, iter_num: 970, loss: 2.3761, 38.80%\n",
      "epoth: 0, iter_num: 980, loss: 2.2756, 39.20%\n",
      "epoth: 0, iter_num: 990, loss: 2.1857, 39.60%\n",
      "epoth: 0, iter_num: 1000, loss: 2.1257, 40.00%\n",
      "epoth: 0, iter_num: 1010, loss: 2.1944, 40.40%\n",
      "epoth: 0, iter_num: 1020, loss: 2.4697, 40.80%\n",
      "epoth: 0, iter_num: 1030, loss: 2.2780, 41.20%\n",
      "epoth: 0, iter_num: 1040, loss: 2.4591, 41.60%\n",
      "epoth: 0, iter_num: 1050, loss: 2.3880, 42.00%\n",
      "epoth: 0, iter_num: 1060, loss: 2.5031, 42.40%\n",
      "epoth: 0, iter_num: 1070, loss: 2.2055, 42.80%\n",
      "epoth: 0, iter_num: 1080, loss: 2.2182, 43.20%\n",
      "epoth: 0, iter_num: 1090, loss: 1.9972, 43.60%\n",
      "epoth: 0, iter_num: 1100, loss: 2.3760, 44.00%\n",
      "epoth: 0, iter_num: 1110, loss: 2.2640, 44.40%\n",
      "epoth: 0, iter_num: 1120, loss: 2.2170, 44.80%\n",
      "epoth: 0, iter_num: 1130, loss: 2.1152, 45.20%\n",
      "epoth: 0, iter_num: 1140, loss: 2.2154, 45.60%\n",
      "epoth: 0, iter_num: 1150, loss: 2.2613, 46.00%\n",
      "epoth: 0, iter_num: 1160, loss: 2.0263, 46.40%\n",
      "epoth: 0, iter_num: 1170, loss: 2.2537, 46.80%\n",
      "epoth: 0, iter_num: 1180, loss: 2.3330, 47.20%\n",
      "epoth: 0, iter_num: 1190, loss: 2.0851, 47.60%\n",
      "epoth: 0, iter_num: 1200, loss: 2.0722, 48.00%\n",
      "epoth: 0, iter_num: 1210, loss: 2.2814, 48.40%\n",
      "epoth: 0, iter_num: 1220, loss: 1.8456, 48.80%\n",
      "epoth: 0, iter_num: 1230, loss: 2.1138, 49.20%\n",
      "epoth: 0, iter_num: 1240, loss: 2.2149, 49.60%\n",
      "epoth: 0, iter_num: 1250, loss: 2.1048, 50.00%\n",
      "epoth: 0, iter_num: 1260, loss: 2.2115, 50.40%\n",
      "epoth: 0, iter_num: 1270, loss: 1.8728, 50.80%\n",
      "epoth: 0, iter_num: 1280, loss: 2.0608, 51.20%\n",
      "epoth: 0, iter_num: 1290, loss: 2.0894, 51.60%\n",
      "epoth: 0, iter_num: 1300, loss: 1.9266, 52.00%\n",
      "epoth: 0, iter_num: 1310, loss: 1.8891, 52.40%\n",
      "epoth: 0, iter_num: 1320, loss: 1.6565, 52.80%\n",
      "epoth: 0, iter_num: 1330, loss: 1.7962, 53.20%\n",
      "epoth: 0, iter_num: 1340, loss: 1.9376, 53.60%\n",
      "epoth: 0, iter_num: 1350, loss: 2.0293, 54.00%\n",
      "epoth: 0, iter_num: 1360, loss: 2.0469, 54.40%\n",
      "epoth: 0, iter_num: 1370, loss: 1.8357, 54.80%\n",
      "epoth: 0, iter_num: 1380, loss: 1.7806, 55.20%\n",
      "epoth: 0, iter_num: 1390, loss: 2.0893, 55.60%\n",
      "epoth: 0, iter_num: 1400, loss: 2.0341, 56.00%\n",
      "epoth: 0, iter_num: 1410, loss: 1.9933, 56.40%\n",
      "epoth: 0, iter_num: 1420, loss: 1.7494, 56.80%\n",
      "epoth: 0, iter_num: 1430, loss: 1.8160, 57.20%\n",
      "epoth: 0, iter_num: 1440, loss: 2.0783, 57.60%\n",
      "epoth: 0, iter_num: 1450, loss: 1.9125, 58.00%\n",
      "epoth: 0, iter_num: 1460, loss: 2.1150, 58.40%\n",
      "epoth: 0, iter_num: 1470, loss: 1.7117, 58.80%\n",
      "epoth: 0, iter_num: 1480, loss: 1.9189, 59.20%\n",
      "epoth: 0, iter_num: 1490, loss: 2.3356, 59.60%\n",
      "epoth: 0, iter_num: 1500, loss: 2.0338, 60.00%\n",
      "epoth: 0, iter_num: 1510, loss: 2.1315, 60.40%\n",
      "epoth: 0, iter_num: 1520, loss: 2.1368, 60.80%\n",
      "epoth: 0, iter_num: 1530, loss: 1.9103, 61.20%\n",
      "epoth: 0, iter_num: 1540, loss: 1.8323, 61.60%\n",
      "epoth: 0, iter_num: 1550, loss: 1.9375, 62.00%\n",
      "epoth: 0, iter_num: 1560, loss: 1.6925, 62.40%\n",
      "epoth: 0, iter_num: 1570, loss: 1.7491, 62.80%\n",
      "epoth: 0, iter_num: 1580, loss: 1.8544, 63.20%\n",
      "epoth: 0, iter_num: 1590, loss: 1.7870, 63.60%\n",
      "epoth: 0, iter_num: 1600, loss: 1.6745, 64.00%\n",
      "epoth: 0, iter_num: 1610, loss: 1.6000, 64.40%\n",
      "epoth: 0, iter_num: 1620, loss: 1.8171, 64.80%\n",
      "epoth: 0, iter_num: 1630, loss: 1.7594, 65.20%\n",
      "epoth: 0, iter_num: 1640, loss: 1.7340, 65.60%\n",
      "epoth: 0, iter_num: 1650, loss: 1.7046, 66.00%\n",
      "epoth: 0, iter_num: 1660, loss: 2.0193, 66.40%\n",
      "epoth: 0, iter_num: 1670, loss: 1.8255, 66.80%\n",
      "epoth: 0, iter_num: 1680, loss: 1.8343, 67.20%\n",
      "epoth: 0, iter_num: 1690, loss: 2.0119, 67.60%\n",
      "epoth: 0, iter_num: 1700, loss: 1.6806, 68.00%\n",
      "epoth: 0, iter_num: 1710, loss: 1.8465, 68.40%\n",
      "epoth: 0, iter_num: 1720, loss: 1.8643, 68.80%\n",
      "epoth: 0, iter_num: 1730, loss: 1.8772, 69.20%\n",
      "epoth: 0, iter_num: 1740, loss: 1.9485, 69.60%\n",
      "epoth: 0, iter_num: 1750, loss: 1.5995, 70.00%\n",
      "epoth: 0, iter_num: 1760, loss: 1.6663, 70.40%\n",
      "epoth: 0, iter_num: 1770, loss: 1.7978, 70.80%\n",
      "epoth: 0, iter_num: 1780, loss: 1.8382, 71.20%\n",
      "epoth: 0, iter_num: 1790, loss: 2.0777, 71.60%\n",
      "epoth: 0, iter_num: 1800, loss: 1.8101, 72.00%\n",
      "epoth: 0, iter_num: 1810, loss: 1.9091, 72.40%\n",
      "epoth: 0, iter_num: 1820, loss: 2.0486, 72.80%\n",
      "epoth: 0, iter_num: 1830, loss: 1.5673, 73.20%\n",
      "epoth: 0, iter_num: 1840, loss: 1.6866, 73.60%\n",
      "epoth: 0, iter_num: 1850, loss: 1.3701, 74.00%\n",
      "epoth: 0, iter_num: 1860, loss: 2.0375, 74.40%\n",
      "epoth: 0, iter_num: 1870, loss: 1.7231, 74.80%\n",
      "epoth: 0, iter_num: 1880, loss: 1.5611, 75.20%\n",
      "epoth: 0, iter_num: 1890, loss: 1.5749, 75.60%\n",
      "epoth: 0, iter_num: 1900, loss: 1.6891, 76.00%\n",
      "epoth: 0, iter_num: 1910, loss: 1.7545, 76.40%\n",
      "epoth: 0, iter_num: 1920, loss: 1.6163, 76.80%\n",
      "epoth: 0, iter_num: 1930, loss: 1.8944, 77.20%\n",
      "epoth: 0, iter_num: 1940, loss: 2.0365, 77.60%\n",
      "epoth: 0, iter_num: 1950, loss: 1.2162, 78.00%\n",
      "epoth: 0, iter_num: 1960, loss: 1.3949, 78.40%\n",
      "epoth: 0, iter_num: 1970, loss: 1.4807, 78.80%\n",
      "epoth: 0, iter_num: 1980, loss: 1.4341, 79.20%\n",
      "epoth: 0, iter_num: 1990, loss: 1.5933, 79.60%\n",
      "epoth: 0, iter_num: 2000, loss: 1.4742, 80.00%\n",
      "epoth: 0, iter_num: 2010, loss: 1.3956, 80.40%\n",
      "epoth: 0, iter_num: 2020, loss: 1.6462, 80.80%\n",
      "epoth: 0, iter_num: 2030, loss: 1.9276, 81.20%\n",
      "epoth: 0, iter_num: 2040, loss: 1.8598, 81.60%\n",
      "epoth: 0, iter_num: 2050, loss: 1.6998, 82.00%\n",
      "epoth: 0, iter_num: 2060, loss: 1.5141, 82.40%\n",
      "epoth: 0, iter_num: 2070, loss: 1.5587, 82.80%\n",
      "epoth: 0, iter_num: 2080, loss: 1.3889, 83.20%\n",
      "epoth: 0, iter_num: 2090, loss: 1.6521, 83.60%\n",
      "epoth: 0, iter_num: 2100, loss: 1.1957, 84.00%\n",
      "epoth: 0, iter_num: 2110, loss: 1.8467, 84.40%\n",
      "epoth: 0, iter_num: 2120, loss: 1.9400, 84.80%\n",
      "epoth: 0, iter_num: 2130, loss: 1.6549, 85.20%\n",
      "epoth: 0, iter_num: 2140, loss: 1.4175, 85.60%\n",
      "epoth: 0, iter_num: 2150, loss: 1.2965, 86.00%\n",
      "epoth: 0, iter_num: 2160, loss: 1.3501, 86.40%\n",
      "epoth: 0, iter_num: 2170, loss: 1.4062, 86.80%\n",
      "epoth: 0, iter_num: 2180, loss: 1.6402, 87.20%\n",
      "epoth: 0, iter_num: 2190, loss: 1.5981, 87.60%\n",
      "epoth: 0, iter_num: 2200, loss: 1.4801, 88.00%\n",
      "epoth: 0, iter_num: 2210, loss: 1.7716, 88.40%\n",
      "epoth: 0, iter_num: 2220, loss: 1.8054, 88.80%\n",
      "epoth: 0, iter_num: 2230, loss: 1.8252, 89.20%\n",
      "epoth: 0, iter_num: 2240, loss: 1.6178, 89.60%\n",
      "epoth: 0, iter_num: 2250, loss: 1.4994, 90.00%\n",
      "epoth: 0, iter_num: 2260, loss: 1.1295, 90.40%\n",
      "epoth: 0, iter_num: 2270, loss: 1.5950, 90.80%\n",
      "epoth: 0, iter_num: 2280, loss: 1.4665, 91.20%\n",
      "epoth: 0, iter_num: 2290, loss: 1.4670, 91.60%\n",
      "epoth: 0, iter_num: 2300, loss: 1.4480, 92.00%\n",
      "epoth: 0, iter_num: 2310, loss: 1.7561, 92.40%\n",
      "epoth: 0, iter_num: 2320, loss: 1.2540, 92.80%\n",
      "epoth: 0, iter_num: 2330, loss: 1.5155, 93.20%\n",
      "epoth: 0, iter_num: 2340, loss: 1.7859, 93.60%\n",
      "epoth: 0, iter_num: 2350, loss: 1.5129, 94.00%\n",
      "epoth: 0, iter_num: 2360, loss: 1.4444, 94.40%\n",
      "epoth: 0, iter_num: 2370, loss: 1.7912, 94.80%\n",
      "epoth: 0, iter_num: 2380, loss: 1.4195, 95.20%\n",
      "epoth: 0, iter_num: 2390, loss: 1.4309, 95.60%\n",
      "epoth: 0, iter_num: 2400, loss: 1.6764, 96.00%\n",
      "epoth: 0, iter_num: 2410, loss: 1.8316, 96.40%\n",
      "epoth: 0, iter_num: 2420, loss: 1.5239, 96.80%\n",
      "epoth: 0, iter_num: 2430, loss: 1.3656, 97.20%\n",
      "epoth: 0, iter_num: 2440, loss: 1.2688, 97.60%\n",
      "epoth: 0, iter_num: 2450, loss: 1.4328, 98.00%\n",
      "epoth: 0, iter_num: 2460, loss: 1.1577, 98.40%\n",
      "epoth: 0, iter_num: 2470, loss: 1.4968, 98.80%\n",
      "epoth: 0, iter_num: 2480, loss: 1.3836, 99.20%\n",
      "epoth: 0, iter_num: 2490, loss: 1.4392, 99.60%\n",
      "epoth: 0, iter_num: 2500, loss: 1.6696, 100.00%\n",
      "Epoch: 0, Average training loss: 2.0992\n",
      "Accuracy: 0.6164\n",
      "Average testing loss: 1.4032\n",
      "-------------------------------\n",
      "------------Epoch: 1 ----------------\n",
      "epoth: 1, iter_num: 10, loss: 1.3090, 0.40%\n",
      "epoth: 1, iter_num: 20, loss: 1.4088, 0.80%\n",
      "epoth: 1, iter_num: 30, loss: 1.5094, 1.20%\n",
      "epoth: 1, iter_num: 40, loss: 1.8335, 1.60%\n",
      "epoth: 1, iter_num: 50, loss: 1.5226, 2.00%\n",
      "epoth: 1, iter_num: 60, loss: 1.4783, 2.40%\n",
      "epoth: 1, iter_num: 70, loss: 1.5109, 2.80%\n",
      "epoth: 1, iter_num: 80, loss: 1.4664, 3.20%\n",
      "epoth: 1, iter_num: 90, loss: 1.2504, 3.60%\n",
      "epoth: 1, iter_num: 100, loss: 1.7268, 4.00%\n",
      "epoth: 1, iter_num: 110, loss: 1.6010, 4.40%\n",
      "epoth: 1, iter_num: 120, loss: 1.5313, 4.80%\n",
      "epoth: 1, iter_num: 130, loss: 1.6273, 5.20%\n",
      "epoth: 1, iter_num: 140, loss: 1.8489, 5.60%\n",
      "epoth: 1, iter_num: 150, loss: 1.6270, 6.00%\n",
      "epoth: 1, iter_num: 160, loss: 1.3158, 6.40%\n",
      "epoth: 1, iter_num: 170, loss: 1.3090, 6.80%\n",
      "epoth: 1, iter_num: 180, loss: 1.8758, 7.20%\n",
      "epoth: 1, iter_num: 190, loss: 1.3104, 7.60%\n",
      "epoth: 1, iter_num: 200, loss: 1.3358, 8.00%\n",
      "epoth: 1, iter_num: 210, loss: 1.5597, 8.40%\n",
      "epoth: 1, iter_num: 220, loss: 1.5270, 8.80%\n",
      "epoth: 1, iter_num: 230, loss: 1.4983, 9.20%\n",
      "epoth: 1, iter_num: 240, loss: 1.2460, 9.60%\n",
      "epoth: 1, iter_num: 250, loss: 1.3867, 10.00%\n",
      "epoth: 1, iter_num: 260, loss: 1.5008, 10.40%\n",
      "epoth: 1, iter_num: 270, loss: 1.0974, 10.80%\n",
      "epoth: 1, iter_num: 280, loss: 1.5489, 11.20%\n",
      "epoth: 1, iter_num: 290, loss: 1.6826, 11.60%\n",
      "epoth: 1, iter_num: 300, loss: 1.3919, 12.00%\n",
      "epoth: 1, iter_num: 310, loss: 1.7839, 12.40%\n",
      "epoth: 1, iter_num: 320, loss: 1.5152, 12.80%\n",
      "epoth: 1, iter_num: 330, loss: 1.2084, 13.20%\n",
      "epoth: 1, iter_num: 340, loss: 1.5254, 13.60%\n",
      "epoth: 1, iter_num: 350, loss: 1.5142, 14.00%\n",
      "epoth: 1, iter_num: 360, loss: 1.3324, 14.40%\n",
      "epoth: 1, iter_num: 370, loss: 1.6885, 14.80%\n",
      "epoth: 1, iter_num: 380, loss: 1.5574, 15.20%\n",
      "epoth: 1, iter_num: 390, loss: 1.7551, 15.60%\n",
      "epoth: 1, iter_num: 400, loss: 1.7570, 16.00%\n",
      "epoth: 1, iter_num: 410, loss: 1.6612, 16.40%\n",
      "epoth: 1, iter_num: 420, loss: 1.4145, 16.80%\n",
      "epoth: 1, iter_num: 430, loss: 1.3632, 17.20%\n",
      "epoth: 1, iter_num: 440, loss: 1.3201, 17.60%\n",
      "epoth: 1, iter_num: 450, loss: 1.9160, 18.00%\n",
      "epoth: 1, iter_num: 460, loss: 1.3377, 18.40%\n",
      "epoth: 1, iter_num: 470, loss: 1.3943, 18.80%\n",
      "epoth: 1, iter_num: 480, loss: 1.5616, 19.20%\n",
      "epoth: 1, iter_num: 490, loss: 1.5628, 19.60%\n",
      "epoth: 1, iter_num: 500, loss: 1.1644, 20.00%\n",
      "epoth: 1, iter_num: 510, loss: 1.6847, 20.40%\n",
      "epoth: 1, iter_num: 520, loss: 1.1809, 20.80%\n",
      "epoth: 1, iter_num: 530, loss: 1.3808, 21.20%\n",
      "epoth: 1, iter_num: 540, loss: 1.1589, 21.60%\n",
      "epoth: 1, iter_num: 550, loss: 1.9394, 22.00%\n",
      "epoth: 1, iter_num: 560, loss: 1.6936, 22.40%\n",
      "epoth: 1, iter_num: 570, loss: 1.3855, 22.80%\n",
      "epoth: 1, iter_num: 580, loss: 1.5484, 23.20%\n",
      "epoth: 1, iter_num: 590, loss: 1.5037, 23.60%\n",
      "epoth: 1, iter_num: 600, loss: 1.2798, 24.00%\n",
      "epoth: 1, iter_num: 610, loss: 1.8121, 24.40%\n",
      "epoth: 1, iter_num: 620, loss: 1.7371, 24.80%\n",
      "epoth: 1, iter_num: 630, loss: 1.6493, 25.20%\n",
      "epoth: 1, iter_num: 640, loss: 1.7354, 25.60%\n",
      "epoth: 1, iter_num: 650, loss: 1.9190, 26.00%\n",
      "epoth: 1, iter_num: 660, loss: 1.4031, 26.40%\n",
      "epoth: 1, iter_num: 670, loss: 1.8367, 26.80%\n",
      "epoth: 1, iter_num: 680, loss: 1.1450, 27.20%\n",
      "epoth: 1, iter_num: 690, loss: 1.6295, 27.60%\n",
      "epoth: 1, iter_num: 700, loss: 1.6040, 28.00%\n",
      "epoth: 1, iter_num: 710, loss: 1.2534, 28.40%\n",
      "epoth: 1, iter_num: 720, loss: 1.7092, 28.80%\n",
      "epoth: 1, iter_num: 730, loss: 2.0051, 29.20%\n",
      "epoth: 1, iter_num: 740, loss: 1.5893, 29.60%\n",
      "epoth: 1, iter_num: 750, loss: 1.8211, 30.00%\n",
      "epoth: 1, iter_num: 760, loss: 1.1857, 30.40%\n",
      "epoth: 1, iter_num: 770, loss: 1.6226, 30.80%\n",
      "epoth: 1, iter_num: 780, loss: 1.3020, 31.20%\n",
      "epoth: 1, iter_num: 790, loss: 1.6658, 31.60%\n",
      "epoth: 1, iter_num: 800, loss: 1.9028, 32.00%\n",
      "epoth: 1, iter_num: 810, loss: 1.3710, 32.40%\n",
      "epoth: 1, iter_num: 820, loss: 1.5235, 32.80%\n",
      "epoth: 1, iter_num: 830, loss: 1.4068, 33.20%\n",
      "epoth: 1, iter_num: 840, loss: 1.4653, 33.60%\n",
      "epoth: 1, iter_num: 850, loss: 1.3627, 34.00%\n",
      "epoth: 1, iter_num: 860, loss: 1.7780, 34.40%\n",
      "epoth: 1, iter_num: 870, loss: 1.1191, 34.80%\n",
      "epoth: 1, iter_num: 880, loss: 1.3874, 35.20%\n",
      "epoth: 1, iter_num: 890, loss: 1.4309, 35.60%\n",
      "epoth: 1, iter_num: 900, loss: 1.3976, 36.00%\n",
      "epoth: 1, iter_num: 910, loss: 1.4438, 36.40%\n",
      "epoth: 1, iter_num: 920, loss: 1.4546, 36.80%\n",
      "epoth: 1, iter_num: 930, loss: 1.5525, 37.20%\n",
      "epoth: 1, iter_num: 940, loss: 1.6842, 37.60%\n",
      "epoth: 1, iter_num: 950, loss: 1.4590, 38.00%\n",
      "epoth: 1, iter_num: 960, loss: 1.4099, 38.40%\n",
      "epoth: 1, iter_num: 970, loss: 2.0705, 38.80%\n",
      "epoth: 1, iter_num: 980, loss: 1.3655, 39.20%\n",
      "epoth: 1, iter_num: 990, loss: 1.6947, 39.60%\n",
      "epoth: 1, iter_num: 1000, loss: 1.3105, 40.00%\n",
      "epoth: 1, iter_num: 1010, loss: 1.0887, 40.40%\n",
      "epoth: 1, iter_num: 1020, loss: 1.6172, 40.80%\n",
      "epoth: 1, iter_num: 1030, loss: 1.6038, 41.20%\n",
      "epoth: 1, iter_num: 1040, loss: 1.5154, 41.60%\n",
      "epoth: 1, iter_num: 1050, loss: 1.2745, 42.00%\n",
      "epoth: 1, iter_num: 1060, loss: 1.3624, 42.40%\n",
      "epoth: 1, iter_num: 1070, loss: 1.5494, 42.80%\n",
      "epoth: 1, iter_num: 1080, loss: 1.3431, 43.20%\n",
      "epoth: 1, iter_num: 1090, loss: 1.5710, 43.60%\n",
      "epoth: 1, iter_num: 1100, loss: 1.3930, 44.00%\n",
      "epoth: 1, iter_num: 1110, loss: 1.5249, 44.40%\n",
      "epoth: 1, iter_num: 1120, loss: 1.7135, 44.80%\n",
      "epoth: 1, iter_num: 1130, loss: 1.7116, 45.20%\n",
      "epoth: 1, iter_num: 1140, loss: 1.4373, 45.60%\n",
      "epoth: 1, iter_num: 1150, loss: 1.3485, 46.00%\n",
      "epoth: 1, iter_num: 1160, loss: 1.6663, 46.40%\n",
      "epoth: 1, iter_num: 1170, loss: 1.8032, 46.80%\n",
      "epoth: 1, iter_num: 1180, loss: 1.4473, 47.20%\n",
      "epoth: 1, iter_num: 1190, loss: 1.0937, 47.60%\n",
      "epoth: 1, iter_num: 1200, loss: 1.2058, 48.00%\n",
      "epoth: 1, iter_num: 1210, loss: 1.7699, 48.40%\n",
      "epoth: 1, iter_num: 1220, loss: 1.3408, 48.80%\n",
      "epoth: 1, iter_num: 1230, loss: 1.6825, 49.20%\n",
      "epoth: 1, iter_num: 1240, loss: 1.5390, 49.60%\n",
      "epoth: 1, iter_num: 1250, loss: 1.3231, 50.00%\n",
      "epoth: 1, iter_num: 1260, loss: 1.4941, 50.40%\n",
      "epoth: 1, iter_num: 1270, loss: 1.5719, 50.80%\n",
      "epoth: 1, iter_num: 1280, loss: 1.7751, 51.20%\n",
      "epoth: 1, iter_num: 1290, loss: 1.7137, 51.60%\n",
      "epoth: 1, iter_num: 1300, loss: 1.4966, 52.00%\n",
      "epoth: 1, iter_num: 1310, loss: 1.2652, 52.40%\n",
      "epoth: 1, iter_num: 1320, loss: 1.6356, 52.80%\n",
      "epoth: 1, iter_num: 1330, loss: 1.0964, 53.20%\n",
      "epoth: 1, iter_num: 1340, loss: 1.4214, 53.60%\n",
      "epoth: 1, iter_num: 1350, loss: 1.2387, 54.00%\n",
      "epoth: 1, iter_num: 1360, loss: 1.7350, 54.40%\n",
      "epoth: 1, iter_num: 1370, loss: 1.3624, 54.80%\n",
      "epoth: 1, iter_num: 1380, loss: 1.5061, 55.20%\n",
      "epoth: 1, iter_num: 1390, loss: 1.5205, 55.60%\n",
      "epoth: 1, iter_num: 1400, loss: 1.3040, 56.00%\n",
      "epoth: 1, iter_num: 1410, loss: 1.2538, 56.40%\n",
      "epoth: 1, iter_num: 1420, loss: 1.5421, 56.80%\n",
      "epoth: 1, iter_num: 1430, loss: 1.5042, 57.20%\n",
      "epoth: 1, iter_num: 1440, loss: 1.4026, 57.60%\n",
      "epoth: 1, iter_num: 1450, loss: 1.9462, 58.00%\n",
      "epoth: 1, iter_num: 1460, loss: 1.9513, 58.40%\n",
      "epoth: 1, iter_num: 1470, loss: 1.6731, 58.80%\n",
      "epoth: 1, iter_num: 1480, loss: 1.5989, 59.20%\n",
      "epoth: 1, iter_num: 1490, loss: 1.6941, 59.60%\n",
      "epoth: 1, iter_num: 1500, loss: 1.6495, 60.00%\n",
      "epoth: 1, iter_num: 1510, loss: 1.4621, 60.40%\n",
      "epoth: 1, iter_num: 1520, loss: 1.2929, 60.80%\n",
      "epoth: 1, iter_num: 1530, loss: 1.7416, 61.20%\n",
      "epoth: 1, iter_num: 1540, loss: 1.6387, 61.60%\n",
      "epoth: 1, iter_num: 1550, loss: 1.3775, 62.00%\n",
      "epoth: 1, iter_num: 1560, loss: 1.1947, 62.40%\n",
      "epoth: 1, iter_num: 1570, loss: 1.3915, 62.80%\n",
      "epoth: 1, iter_num: 1580, loss: 1.3434, 63.20%\n",
      "epoth: 1, iter_num: 1590, loss: 1.3011, 63.60%\n",
      "epoth: 1, iter_num: 1600, loss: 1.5423, 64.00%\n",
      "epoth: 1, iter_num: 1610, loss: 1.6762, 64.40%\n",
      "epoth: 1, iter_num: 1620, loss: 1.8668, 64.80%\n",
      "epoth: 1, iter_num: 1630, loss: 1.2711, 65.20%\n",
      "epoth: 1, iter_num: 1640, loss: 1.5002, 65.60%\n",
      "epoth: 1, iter_num: 1650, loss: 1.3724, 66.00%\n",
      "epoth: 1, iter_num: 1660, loss: 1.4975, 66.40%\n",
      "epoth: 1, iter_num: 1670, loss: 1.9527, 66.80%\n",
      "epoth: 1, iter_num: 1680, loss: 1.2868, 67.20%\n",
      "epoth: 1, iter_num: 1690, loss: 1.7970, 67.60%\n",
      "epoth: 1, iter_num: 1700, loss: 1.5171, 68.00%\n",
      "epoth: 1, iter_num: 1710, loss: 1.4125, 68.40%\n",
      "epoth: 1, iter_num: 1720, loss: 1.7307, 68.80%\n",
      "epoth: 1, iter_num: 1730, loss: 1.2945, 69.20%\n",
      "epoth: 1, iter_num: 1740, loss: 1.4452, 69.60%\n",
      "epoth: 1, iter_num: 1750, loss: 1.5555, 70.00%\n",
      "epoth: 1, iter_num: 1760, loss: 1.4690, 70.40%\n",
      "epoth: 1, iter_num: 1770, loss: 1.5371, 70.80%\n",
      "epoth: 1, iter_num: 1780, loss: 1.5122, 71.20%\n",
      "epoth: 1, iter_num: 1790, loss: 1.5638, 71.60%\n",
      "epoth: 1, iter_num: 1800, loss: 1.2070, 72.00%\n",
      "epoth: 1, iter_num: 1810, loss: 1.7976, 72.40%\n",
      "epoth: 1, iter_num: 1820, loss: 1.5975, 72.80%\n",
      "epoth: 1, iter_num: 1830, loss: 1.5647, 73.20%\n",
      "epoth: 1, iter_num: 1840, loss: 1.7299, 73.60%\n",
      "epoth: 1, iter_num: 1850, loss: 1.7375, 74.00%\n",
      "epoth: 1, iter_num: 1860, loss: 1.4664, 74.40%\n",
      "epoth: 1, iter_num: 1870, loss: 1.8854, 74.80%\n",
      "epoth: 1, iter_num: 1880, loss: 1.4022, 75.20%\n",
      "epoth: 1, iter_num: 1890, loss: 1.3163, 75.60%\n",
      "epoth: 1, iter_num: 1900, loss: 1.9804, 76.00%\n",
      "epoth: 1, iter_num: 1910, loss: 1.4151, 76.40%\n",
      "epoth: 1, iter_num: 1920, loss: 1.2520, 76.80%\n",
      "epoth: 1, iter_num: 1930, loss: 1.7425, 77.20%\n",
      "epoth: 1, iter_num: 1940, loss: 1.2579, 77.60%\n",
      "epoth: 1, iter_num: 1950, loss: 2.0994, 78.00%\n",
      "epoth: 1, iter_num: 1960, loss: 1.6508, 78.40%\n",
      "epoth: 1, iter_num: 1970, loss: 1.4785, 78.80%\n",
      "epoth: 1, iter_num: 1980, loss: 1.9048, 79.20%\n",
      "epoth: 1, iter_num: 1990, loss: 1.3188, 79.60%\n",
      "epoth: 1, iter_num: 2000, loss: 2.2029, 80.00%\n",
      "epoth: 1, iter_num: 2010, loss: 1.6990, 80.40%\n",
      "epoth: 1, iter_num: 2020, loss: 1.3280, 80.80%\n",
      "epoth: 1, iter_num: 2030, loss: 1.5711, 81.20%\n",
      "epoth: 1, iter_num: 2040, loss: 1.3177, 81.60%\n",
      "epoth: 1, iter_num: 2050, loss: 1.5175, 82.00%\n",
      "epoth: 1, iter_num: 2060, loss: 1.5434, 82.40%\n",
      "epoth: 1, iter_num: 2070, loss: 1.4459, 82.80%\n",
      "epoth: 1, iter_num: 2080, loss: 1.3624, 83.20%\n",
      "epoth: 1, iter_num: 2090, loss: 1.4053, 83.60%\n",
      "epoth: 1, iter_num: 2100, loss: 1.0455, 84.00%\n",
      "epoth: 1, iter_num: 2110, loss: 1.5113, 84.40%\n",
      "epoth: 1, iter_num: 2120, loss: 1.9395, 84.80%\n",
      "epoth: 1, iter_num: 2130, loss: 1.3019, 85.20%\n",
      "epoth: 1, iter_num: 2140, loss: 1.1756, 85.60%\n",
      "epoth: 1, iter_num: 2150, loss: 1.3842, 86.00%\n",
      "epoth: 1, iter_num: 2160, loss: 1.4924, 86.40%\n",
      "epoth: 1, iter_num: 2170, loss: 1.4864, 86.80%\n",
      "epoth: 1, iter_num: 2180, loss: 1.2448, 87.20%\n",
      "epoth: 1, iter_num: 2190, loss: 1.2221, 87.60%\n",
      "epoth: 1, iter_num: 2200, loss: 1.5522, 88.00%\n",
      "epoth: 1, iter_num: 2210, loss: 1.7267, 88.40%\n",
      "epoth: 1, iter_num: 2220, loss: 1.4736, 88.80%\n",
      "epoth: 1, iter_num: 2230, loss: 1.6334, 89.20%\n",
      "epoth: 1, iter_num: 2240, loss: 1.4858, 89.60%\n",
      "epoth: 1, iter_num: 2250, loss: 1.6112, 90.00%\n",
      "epoth: 1, iter_num: 2260, loss: 1.0639, 90.40%\n",
      "epoth: 1, iter_num: 2270, loss: 1.6172, 90.80%\n",
      "epoth: 1, iter_num: 2280, loss: 1.1088, 91.20%\n",
      "epoth: 1, iter_num: 2290, loss: 1.3599, 91.60%\n",
      "epoth: 1, iter_num: 2300, loss: 1.0748, 92.00%\n",
      "epoth: 1, iter_num: 2310, loss: 0.9673, 92.40%\n",
      "epoth: 1, iter_num: 2320, loss: 1.2350, 92.80%\n",
      "epoth: 1, iter_num: 2330, loss: 1.2808, 93.20%\n",
      "epoth: 1, iter_num: 2340, loss: 1.0797, 93.60%\n",
      "epoth: 1, iter_num: 2350, loss: 1.6024, 94.00%\n",
      "epoth: 1, iter_num: 2360, loss: 1.2754, 94.40%\n",
      "epoth: 1, iter_num: 2370, loss: 1.2374, 94.80%\n",
      "epoth: 1, iter_num: 2380, loss: 1.1597, 95.20%\n",
      "epoth: 1, iter_num: 2390, loss: 1.3809, 95.60%\n",
      "epoth: 1, iter_num: 2400, loss: 1.4862, 96.00%\n",
      "epoth: 1, iter_num: 2410, loss: 2.0558, 96.40%\n",
      "epoth: 1, iter_num: 2420, loss: 1.5022, 96.80%\n",
      "epoth: 1, iter_num: 2430, loss: 2.0428, 97.20%\n",
      "epoth: 1, iter_num: 2440, loss: 1.6144, 97.60%\n",
      "epoth: 1, iter_num: 2450, loss: 1.1538, 98.00%\n",
      "epoth: 1, iter_num: 2460, loss: 1.0047, 98.40%\n",
      "epoth: 1, iter_num: 2470, loss: 1.2334, 98.80%\n",
      "epoth: 1, iter_num: 2480, loss: 1.3626, 99.20%\n",
      "epoth: 1, iter_num: 2490, loss: 1.9732, 99.60%\n",
      "epoth: 1, iter_num: 2500, loss: 1.7315, 100.00%\n",
      "Epoch: 1, Average training loss: 1.5030\n",
      "Accuracy: 0.6164\n",
      "Average testing loss: 1.4032\n",
      "-------------------------------\n",
      "------------Epoch: 2 ----------------\n",
      "epoth: 2, iter_num: 10, loss: 1.3038, 0.40%\n",
      "epoth: 2, iter_num: 20, loss: 1.4445, 0.80%\n",
      "epoth: 2, iter_num: 30, loss: 1.3006, 1.20%\n",
      "epoth: 2, iter_num: 40, loss: 1.5535, 1.60%\n",
      "epoth: 2, iter_num: 50, loss: 1.5989, 2.00%\n",
      "epoth: 2, iter_num: 60, loss: 1.4526, 2.40%\n",
      "epoth: 2, iter_num: 70, loss: 1.4194, 2.80%\n",
      "epoth: 2, iter_num: 80, loss: 1.2953, 3.20%\n",
      "epoth: 2, iter_num: 90, loss: 1.7710, 3.60%\n",
      "epoth: 2, iter_num: 100, loss: 1.3252, 4.00%\n",
      "epoth: 2, iter_num: 110, loss: 1.6456, 4.40%\n",
      "epoth: 2, iter_num: 120, loss: 1.5918, 4.80%\n",
      "epoth: 2, iter_num: 130, loss: 1.4072, 5.20%\n",
      "epoth: 2, iter_num: 140, loss: 1.2484, 5.60%\n",
      "epoth: 2, iter_num: 150, loss: 1.1750, 6.00%\n",
      "epoth: 2, iter_num: 160, loss: 1.8506, 6.40%\n",
      "epoth: 2, iter_num: 170, loss: 1.7139, 6.80%\n",
      "epoth: 2, iter_num: 180, loss: 1.4987, 7.20%\n",
      "epoth: 2, iter_num: 190, loss: 1.4541, 7.60%\n",
      "epoth: 2, iter_num: 200, loss: 1.7355, 8.00%\n",
      "epoth: 2, iter_num: 210, loss: 1.3083, 8.40%\n",
      "epoth: 2, iter_num: 220, loss: 1.4655, 8.80%\n",
      "epoth: 2, iter_num: 230, loss: 1.4541, 9.20%\n",
      "epoth: 2, iter_num: 240, loss: 1.3197, 9.60%\n",
      "epoth: 2, iter_num: 250, loss: 1.2572, 10.00%\n",
      "epoth: 2, iter_num: 260, loss: 1.5459, 10.40%\n",
      "epoth: 2, iter_num: 270, loss: 1.2675, 10.80%\n",
      "epoth: 2, iter_num: 280, loss: 1.5635, 11.20%\n",
      "epoth: 2, iter_num: 290, loss: 1.5301, 11.60%\n",
      "epoth: 2, iter_num: 300, loss: 1.4009, 12.00%\n",
      "epoth: 2, iter_num: 310, loss: 1.4581, 12.40%\n",
      "epoth: 2, iter_num: 320, loss: 1.3236, 12.80%\n",
      "epoth: 2, iter_num: 330, loss: 1.6311, 13.20%\n",
      "epoth: 2, iter_num: 340, loss: 1.9246, 13.60%\n",
      "epoth: 2, iter_num: 350, loss: 1.8525, 14.00%\n",
      "epoth: 2, iter_num: 360, loss: 1.7842, 14.40%\n",
      "epoth: 2, iter_num: 370, loss: 1.7384, 14.80%\n",
      "epoth: 2, iter_num: 380, loss: 1.4195, 15.20%\n",
      "epoth: 2, iter_num: 390, loss: 1.4147, 15.60%\n",
      "epoth: 2, iter_num: 400, loss: 1.2508, 16.00%\n",
      "epoth: 2, iter_num: 410, loss: 1.3790, 16.40%\n",
      "epoth: 2, iter_num: 420, loss: 1.3999, 16.80%\n",
      "epoth: 2, iter_num: 430, loss: 1.5003, 17.20%\n",
      "epoth: 2, iter_num: 440, loss: 1.4523, 17.60%\n",
      "epoth: 2, iter_num: 450, loss: 1.2731, 18.00%\n",
      "epoth: 2, iter_num: 460, loss: 1.4861, 18.40%\n",
      "epoth: 2, iter_num: 470, loss: 1.2739, 18.80%\n",
      "epoth: 2, iter_num: 480, loss: 1.8667, 19.20%\n",
      "epoth: 2, iter_num: 490, loss: 1.7561, 19.60%\n",
      "epoth: 2, iter_num: 500, loss: 1.7928, 20.00%\n",
      "epoth: 2, iter_num: 510, loss: 1.7658, 20.40%\n",
      "epoth: 2, iter_num: 520, loss: 1.4676, 20.80%\n",
      "epoth: 2, iter_num: 530, loss: 0.8891, 21.20%\n",
      "epoth: 2, iter_num: 540, loss: 1.1160, 21.60%\n",
      "epoth: 2, iter_num: 550, loss: 1.5980, 22.00%\n",
      "epoth: 2, iter_num: 560, loss: 1.4587, 22.40%\n",
      "epoth: 2, iter_num: 570, loss: 1.3753, 22.80%\n",
      "epoth: 2, iter_num: 580, loss: 1.5578, 23.20%\n",
      "epoth: 2, iter_num: 590, loss: 1.7572, 23.60%\n",
      "epoth: 2, iter_num: 600, loss: 1.8827, 24.00%\n",
      "epoth: 2, iter_num: 610, loss: 1.3678, 24.40%\n",
      "epoth: 2, iter_num: 620, loss: 1.7236, 24.80%\n",
      "epoth: 2, iter_num: 630, loss: 1.4615, 25.20%\n",
      "epoth: 2, iter_num: 640, loss: 1.3324, 25.60%\n",
      "epoth: 2, iter_num: 650, loss: 1.4935, 26.00%\n",
      "epoth: 2, iter_num: 660, loss: 1.3131, 26.40%\n",
      "epoth: 2, iter_num: 670, loss: 1.4780, 26.80%\n",
      "epoth: 2, iter_num: 680, loss: 0.9579, 27.20%\n",
      "epoth: 2, iter_num: 690, loss: 1.4568, 27.60%\n",
      "epoth: 2, iter_num: 700, loss: 1.7340, 28.00%\n",
      "epoth: 2, iter_num: 710, loss: 1.4681, 28.40%\n",
      "epoth: 2, iter_num: 720, loss: 1.2117, 28.80%\n",
      "epoth: 2, iter_num: 730, loss: 1.6341, 29.20%\n",
      "epoth: 2, iter_num: 740, loss: 1.5640, 29.60%\n",
      "epoth: 2, iter_num: 750, loss: 1.5726, 30.00%\n",
      "epoth: 2, iter_num: 760, loss: 1.1974, 30.40%\n",
      "epoth: 2, iter_num: 770, loss: 1.6792, 30.80%\n",
      "epoth: 2, iter_num: 780, loss: 1.8875, 31.20%\n",
      "epoth: 2, iter_num: 790, loss: 1.7795, 31.60%\n",
      "epoth: 2, iter_num: 800, loss: 1.1752, 32.00%\n",
      "epoth: 2, iter_num: 810, loss: 1.6780, 32.40%\n",
      "epoth: 2, iter_num: 820, loss: 1.3965, 32.80%\n",
      "epoth: 2, iter_num: 830, loss: 1.1376, 33.20%\n",
      "epoth: 2, iter_num: 840, loss: 1.4953, 33.60%\n",
      "epoth: 2, iter_num: 850, loss: 1.2050, 34.00%\n",
      "epoth: 2, iter_num: 860, loss: 1.5367, 34.40%\n",
      "epoth: 2, iter_num: 870, loss: 1.3650, 34.80%\n",
      "epoth: 2, iter_num: 880, loss: 1.6909, 35.20%\n",
      "epoth: 2, iter_num: 890, loss: 1.2348, 35.60%\n",
      "epoth: 2, iter_num: 900, loss: 1.7453, 36.00%\n",
      "epoth: 2, iter_num: 910, loss: 1.2935, 36.40%\n",
      "epoth: 2, iter_num: 920, loss: 1.2620, 36.80%\n",
      "epoth: 2, iter_num: 930, loss: 1.7065, 37.20%\n",
      "epoth: 2, iter_num: 940, loss: 1.7921, 37.60%\n",
      "epoth: 2, iter_num: 950, loss: 1.2795, 38.00%\n",
      "epoth: 2, iter_num: 960, loss: 1.0988, 38.40%\n",
      "epoth: 2, iter_num: 970, loss: 1.4361, 38.80%\n",
      "epoth: 2, iter_num: 980, loss: 1.4622, 39.20%\n",
      "epoth: 2, iter_num: 990, loss: 1.4101, 39.60%\n",
      "epoth: 2, iter_num: 1000, loss: 1.4240, 40.00%\n",
      "epoth: 2, iter_num: 1010, loss: 1.4338, 40.40%\n",
      "epoth: 2, iter_num: 1020, loss: 1.4225, 40.80%\n",
      "epoth: 2, iter_num: 1030, loss: 1.6371, 41.20%\n",
      "epoth: 2, iter_num: 1040, loss: 1.5875, 41.60%\n",
      "epoth: 2, iter_num: 1050, loss: 1.6916, 42.00%\n",
      "epoth: 2, iter_num: 1060, loss: 1.6964, 42.40%\n",
      "epoth: 2, iter_num: 1070, loss: 1.2920, 42.80%\n",
      "epoth: 2, iter_num: 1080, loss: 1.4261, 43.20%\n",
      "epoth: 2, iter_num: 1090, loss: 1.5688, 43.60%\n",
      "epoth: 2, iter_num: 1100, loss: 1.7438, 44.00%\n",
      "epoth: 2, iter_num: 1110, loss: 1.3786, 44.40%\n",
      "epoth: 2, iter_num: 1120, loss: 1.4175, 44.80%\n",
      "epoth: 2, iter_num: 1130, loss: 1.2248, 45.20%\n",
      "epoth: 2, iter_num: 1140, loss: 1.5676, 45.60%\n",
      "epoth: 2, iter_num: 1150, loss: 1.2799, 46.00%\n",
      "epoth: 2, iter_num: 1160, loss: 1.6072, 46.40%\n",
      "epoth: 2, iter_num: 1170, loss: 1.1053, 46.80%\n",
      "epoth: 2, iter_num: 1180, loss: 1.2842, 47.20%\n",
      "epoth: 2, iter_num: 1190, loss: 1.3240, 47.60%\n",
      "epoth: 2, iter_num: 1200, loss: 1.3595, 48.00%\n",
      "epoth: 2, iter_num: 1210, loss: 1.2796, 48.40%\n",
      "epoth: 2, iter_num: 1220, loss: 1.8786, 48.80%\n",
      "epoth: 2, iter_num: 1230, loss: 1.6393, 49.20%\n",
      "epoth: 2, iter_num: 1240, loss: 0.8997, 49.60%\n",
      "epoth: 2, iter_num: 1250, loss: 1.4062, 50.00%\n",
      "epoth: 2, iter_num: 1260, loss: 1.7711, 50.40%\n",
      "epoth: 2, iter_num: 1270, loss: 1.6078, 50.80%\n",
      "epoth: 2, iter_num: 1280, loss: 1.5162, 51.20%\n",
      "epoth: 2, iter_num: 1290, loss: 1.3615, 51.60%\n",
      "epoth: 2, iter_num: 1300, loss: 1.5102, 52.00%\n",
      "epoth: 2, iter_num: 1310, loss: 1.2540, 52.40%\n",
      "epoth: 2, iter_num: 1320, loss: 1.7628, 52.80%\n",
      "epoth: 2, iter_num: 1330, loss: 1.2903, 53.20%\n",
      "epoth: 2, iter_num: 1340, loss: 1.5647, 53.60%\n",
      "epoth: 2, iter_num: 1350, loss: 1.8676, 54.00%\n",
      "epoth: 2, iter_num: 1360, loss: 1.6723, 54.40%\n",
      "epoth: 2, iter_num: 1370, loss: 1.8614, 54.80%\n",
      "epoth: 2, iter_num: 1380, loss: 1.3051, 55.20%\n",
      "epoth: 2, iter_num: 1390, loss: 1.5919, 55.60%\n",
      "epoth: 2, iter_num: 1400, loss: 1.5661, 56.00%\n",
      "epoth: 2, iter_num: 1410, loss: 1.5777, 56.40%\n",
      "epoth: 2, iter_num: 1420, loss: 1.3445, 56.80%\n",
      "epoth: 2, iter_num: 1430, loss: 1.7167, 57.20%\n",
      "epoth: 2, iter_num: 1440, loss: 1.3255, 57.60%\n",
      "epoth: 2, iter_num: 1450, loss: 1.4739, 58.00%\n",
      "epoth: 2, iter_num: 1460, loss: 1.0037, 58.40%\n",
      "epoth: 2, iter_num: 1470, loss: 1.2874, 58.80%\n",
      "epoth: 2, iter_num: 1480, loss: 1.7523, 59.20%\n",
      "epoth: 2, iter_num: 1490, loss: 1.4604, 59.60%\n",
      "epoth: 2, iter_num: 1500, loss: 1.6379, 60.00%\n",
      "epoth: 2, iter_num: 1510, loss: 1.7714, 60.40%\n",
      "epoth: 2, iter_num: 1520, loss: 1.7567, 60.80%\n",
      "epoth: 2, iter_num: 1530, loss: 1.4763, 61.20%\n",
      "epoth: 2, iter_num: 1540, loss: 1.9467, 61.60%\n",
      "epoth: 2, iter_num: 1550, loss: 1.1622, 62.00%\n",
      "epoth: 2, iter_num: 1560, loss: 1.6373, 62.40%\n",
      "epoth: 2, iter_num: 1570, loss: 1.6063, 62.80%\n",
      "epoth: 2, iter_num: 1580, loss: 1.5238, 63.20%\n",
      "epoth: 2, iter_num: 1590, loss: 1.4085, 63.60%\n",
      "epoth: 2, iter_num: 1600, loss: 1.9825, 64.00%\n",
      "epoth: 2, iter_num: 1610, loss: 1.3184, 64.40%\n",
      "epoth: 2, iter_num: 1620, loss: 1.6388, 64.80%\n",
      "epoth: 2, iter_num: 1630, loss: 1.9246, 65.20%\n",
      "epoth: 2, iter_num: 1640, loss: 1.7656, 65.60%\n",
      "epoth: 2, iter_num: 1650, loss: 1.2410, 66.00%\n",
      "epoth: 2, iter_num: 1660, loss: 1.1961, 66.40%\n",
      "epoth: 2, iter_num: 1670, loss: 1.6483, 66.80%\n",
      "epoth: 2, iter_num: 1680, loss: 1.4705, 67.20%\n",
      "epoth: 2, iter_num: 1690, loss: 1.5781, 67.60%\n",
      "epoth: 2, iter_num: 1700, loss: 1.0938, 68.00%\n",
      "epoth: 2, iter_num: 1710, loss: 1.6040, 68.40%\n",
      "epoth: 2, iter_num: 1720, loss: 1.3372, 68.80%\n",
      "epoth: 2, iter_num: 1730, loss: 1.7315, 69.20%\n",
      "epoth: 2, iter_num: 1740, loss: 1.5576, 69.60%\n",
      "epoth: 2, iter_num: 1750, loss: 1.3661, 70.00%\n",
      "epoth: 2, iter_num: 1760, loss: 1.2579, 70.40%\n",
      "epoth: 2, iter_num: 1770, loss: 1.6569, 70.80%\n",
      "epoth: 2, iter_num: 1780, loss: 1.5620, 71.20%\n",
      "epoth: 2, iter_num: 1790, loss: 1.2656, 71.60%\n",
      "epoth: 2, iter_num: 1800, loss: 1.0446, 72.00%\n",
      "epoth: 2, iter_num: 1810, loss: 1.6444, 72.40%\n",
      "epoth: 2, iter_num: 1820, loss: 1.5968, 72.80%\n",
      "epoth: 2, iter_num: 1830, loss: 1.4536, 73.20%\n",
      "epoth: 2, iter_num: 1840, loss: 1.1544, 73.60%\n",
      "epoth: 2, iter_num: 1850, loss: 1.6991, 74.00%\n",
      "epoth: 2, iter_num: 1860, loss: 1.3827, 74.40%\n",
      "epoth: 2, iter_num: 1870, loss: 1.5064, 74.80%\n",
      "epoth: 2, iter_num: 1880, loss: 1.0630, 75.20%\n",
      "epoth: 2, iter_num: 1890, loss: 1.0745, 75.60%\n",
      "epoth: 2, iter_num: 1900, loss: 1.2127, 76.00%\n",
      "epoth: 2, iter_num: 1910, loss: 1.5496, 76.40%\n",
      "epoth: 2, iter_num: 1920, loss: 1.3308, 76.80%\n",
      "epoth: 2, iter_num: 1930, loss: 1.6109, 77.20%\n",
      "epoth: 2, iter_num: 1940, loss: 1.7661, 77.60%\n",
      "epoth: 2, iter_num: 1950, loss: 1.7715, 78.00%\n",
      "epoth: 2, iter_num: 1960, loss: 1.5707, 78.40%\n",
      "epoth: 2, iter_num: 1970, loss: 1.6963, 78.80%\n",
      "epoth: 2, iter_num: 1980, loss: 1.5681, 79.20%\n",
      "epoth: 2, iter_num: 1990, loss: 1.5473, 79.60%\n",
      "epoth: 2, iter_num: 2000, loss: 1.7015, 80.00%\n",
      "epoth: 2, iter_num: 2010, loss: 1.8149, 80.40%\n",
      "epoth: 2, iter_num: 2020, loss: 1.7175, 80.80%\n",
      "epoth: 2, iter_num: 2030, loss: 1.4993, 81.20%\n",
      "epoth: 2, iter_num: 2040, loss: 1.1853, 81.60%\n",
      "epoth: 2, iter_num: 2050, loss: 1.4419, 82.00%\n",
      "epoth: 2, iter_num: 2060, loss: 1.3093, 82.40%\n",
      "epoth: 2, iter_num: 2070, loss: 1.4042, 82.80%\n",
      "epoth: 2, iter_num: 2080, loss: 1.5174, 83.20%\n",
      "epoth: 2, iter_num: 2090, loss: 1.5986, 83.60%\n",
      "epoth: 2, iter_num: 2100, loss: 1.7726, 84.00%\n",
      "epoth: 2, iter_num: 2110, loss: 1.6721, 84.40%\n",
      "epoth: 2, iter_num: 2120, loss: 1.3102, 84.80%\n",
      "epoth: 2, iter_num: 2130, loss: 1.5198, 85.20%\n",
      "epoth: 2, iter_num: 2140, loss: 1.3653, 85.60%\n",
      "epoth: 2, iter_num: 2150, loss: 2.0625, 86.00%\n",
      "epoth: 2, iter_num: 2160, loss: 1.7471, 86.40%\n",
      "epoth: 2, iter_num: 2170, loss: 1.8051, 86.80%\n",
      "epoth: 2, iter_num: 2180, loss: 1.1045, 87.20%\n",
      "epoth: 2, iter_num: 2190, loss: 1.7389, 87.60%\n",
      "epoth: 2, iter_num: 2200, loss: 1.7849, 88.00%\n",
      "epoth: 2, iter_num: 2210, loss: 1.6782, 88.40%\n",
      "epoth: 2, iter_num: 2220, loss: 1.2173, 88.80%\n",
      "epoth: 2, iter_num: 2230, loss: 1.4268, 89.20%\n",
      "epoth: 2, iter_num: 2240, loss: 1.1439, 89.60%\n",
      "epoth: 2, iter_num: 2250, loss: 1.7067, 90.00%\n",
      "epoth: 2, iter_num: 2260, loss: 1.1661, 90.40%\n",
      "epoth: 2, iter_num: 2270, loss: 0.8793, 90.80%\n",
      "epoth: 2, iter_num: 2280, loss: 1.4067, 91.20%\n",
      "epoth: 2, iter_num: 2290, loss: 1.1584, 91.60%\n",
      "epoth: 2, iter_num: 2300, loss: 1.1872, 92.00%\n",
      "epoth: 2, iter_num: 2310, loss: 1.1851, 92.40%\n",
      "epoth: 2, iter_num: 2320, loss: 1.0012, 92.80%\n",
      "epoth: 2, iter_num: 2330, loss: 1.5570, 93.20%\n",
      "epoth: 2, iter_num: 2340, loss: 1.6973, 93.60%\n",
      "epoth: 2, iter_num: 2350, loss: 1.1768, 94.00%\n",
      "epoth: 2, iter_num: 2360, loss: 2.0826, 94.40%\n",
      "epoth: 2, iter_num: 2370, loss: 1.3622, 94.80%\n",
      "epoth: 2, iter_num: 2380, loss: 1.4738, 95.20%\n",
      "epoth: 2, iter_num: 2390, loss: 1.7295, 95.60%\n",
      "epoth: 2, iter_num: 2400, loss: 1.4021, 96.00%\n",
      "epoth: 2, iter_num: 2410, loss: 1.2528, 96.40%\n",
      "epoth: 2, iter_num: 2420, loss: 2.1221, 96.80%\n",
      "epoth: 2, iter_num: 2430, loss: 1.3991, 97.20%\n",
      "epoth: 2, iter_num: 2440, loss: 1.2745, 97.60%\n",
      "epoth: 2, iter_num: 2450, loss: 1.5213, 98.00%\n",
      "epoth: 2, iter_num: 2460, loss: 1.6288, 98.40%\n",
      "epoth: 2, iter_num: 2470, loss: 1.6209, 98.80%\n",
      "epoth: 2, iter_num: 2480, loss: 1.9987, 99.20%\n",
      "epoth: 2, iter_num: 2490, loss: 1.3379, 99.60%\n",
      "epoth: 2, iter_num: 2500, loss: 1.5338, 100.00%\n",
      "Epoch: 2, Average training loss: 1.5012\n",
      "Accuracy: 0.6164\n",
      "Average testing loss: 1.4032\n",
      "-------------------------------\n",
      "------------Epoch: 3 ----------------\n",
      "epoth: 3, iter_num: 10, loss: 1.0545, 0.40%\n",
      "epoth: 3, iter_num: 20, loss: 1.7102, 0.80%\n",
      "epoth: 3, iter_num: 30, loss: 1.7880, 1.20%\n",
      "epoth: 3, iter_num: 40, loss: 1.2766, 1.60%\n",
      "epoth: 3, iter_num: 50, loss: 1.6605, 2.00%\n",
      "epoth: 3, iter_num: 60, loss: 1.9883, 2.40%\n",
      "epoth: 3, iter_num: 70, loss: 1.5094, 2.80%\n",
      "epoth: 3, iter_num: 80, loss: 1.5857, 3.20%\n",
      "epoth: 3, iter_num: 90, loss: 1.5060, 3.60%\n",
      "epoth: 3, iter_num: 100, loss: 1.6989, 4.00%\n",
      "epoth: 3, iter_num: 110, loss: 1.0112, 4.40%\n",
      "epoth: 3, iter_num: 120, loss: 1.2713, 4.80%\n",
      "epoth: 3, iter_num: 130, loss: 1.5478, 5.20%\n",
      "epoth: 3, iter_num: 140, loss: 1.2760, 5.60%\n",
      "epoth: 3, iter_num: 150, loss: 1.5329, 6.00%\n",
      "epoth: 3, iter_num: 160, loss: 1.3870, 6.40%\n",
      "epoth: 3, iter_num: 170, loss: 1.8738, 6.80%\n",
      "epoth: 3, iter_num: 180, loss: 1.5052, 7.20%\n",
      "epoth: 3, iter_num: 190, loss: 1.4925, 7.60%\n",
      "epoth: 3, iter_num: 200, loss: 1.6439, 8.00%\n",
      "epoth: 3, iter_num: 210, loss: 1.1205, 8.40%\n",
      "epoth: 3, iter_num: 220, loss: 1.7428, 8.80%\n",
      "epoth: 3, iter_num: 230, loss: 1.6628, 9.20%\n",
      "epoth: 3, iter_num: 240, loss: 1.7023, 9.60%\n",
      "epoth: 3, iter_num: 250, loss: 1.3772, 10.00%\n",
      "epoth: 3, iter_num: 260, loss: 1.1896, 10.40%\n",
      "epoth: 3, iter_num: 270, loss: 1.5302, 10.80%\n",
      "epoth: 3, iter_num: 280, loss: 1.8737, 11.20%\n",
      "epoth: 3, iter_num: 290, loss: 1.1157, 11.60%\n",
      "epoth: 3, iter_num: 300, loss: 1.8051, 12.00%\n",
      "epoth: 3, iter_num: 310, loss: 1.2286, 12.40%\n",
      "epoth: 3, iter_num: 320, loss: 1.5999, 12.80%\n",
      "epoth: 3, iter_num: 330, loss: 1.7675, 13.20%\n",
      "epoth: 3, iter_num: 340, loss: 1.6923, 13.60%\n",
      "epoth: 3, iter_num: 350, loss: 1.0987, 14.00%\n",
      "epoth: 3, iter_num: 360, loss: 1.3293, 14.40%\n",
      "epoth: 3, iter_num: 370, loss: 1.7801, 14.80%\n",
      "epoth: 3, iter_num: 380, loss: 1.3533, 15.20%\n",
      "epoth: 3, iter_num: 390, loss: 1.8063, 15.60%\n",
      "epoth: 3, iter_num: 400, loss: 1.4151, 16.00%\n",
      "epoth: 3, iter_num: 410, loss: 1.4387, 16.40%\n",
      "epoth: 3, iter_num: 420, loss: 1.5710, 16.80%\n",
      "epoth: 3, iter_num: 430, loss: 1.6334, 17.20%\n",
      "epoth: 3, iter_num: 440, loss: 1.1778, 17.60%\n",
      "epoth: 3, iter_num: 450, loss: 2.0863, 18.00%\n",
      "epoth: 3, iter_num: 460, loss: 1.1111, 18.40%\n",
      "epoth: 3, iter_num: 470, loss: 1.4382, 18.80%\n",
      "epoth: 3, iter_num: 480, loss: 1.4569, 19.20%\n",
      "epoth: 3, iter_num: 490, loss: 1.5152, 19.60%\n",
      "epoth: 3, iter_num: 500, loss: 1.6560, 20.00%\n",
      "epoth: 3, iter_num: 510, loss: 1.5322, 20.40%\n",
      "epoth: 3, iter_num: 520, loss: 1.4353, 20.80%\n",
      "epoth: 3, iter_num: 530, loss: 1.3509, 21.20%\n",
      "epoth: 3, iter_num: 540, loss: 1.3528, 21.60%\n",
      "epoth: 3, iter_num: 550, loss: 1.5255, 22.00%\n",
      "epoth: 3, iter_num: 560, loss: 1.9320, 22.40%\n",
      "epoth: 3, iter_num: 570, loss: 1.8485, 22.80%\n",
      "epoth: 3, iter_num: 580, loss: 1.5533, 23.20%\n",
      "epoth: 3, iter_num: 590, loss: 1.6680, 23.60%\n",
      "epoth: 3, iter_num: 600, loss: 1.5687, 24.00%\n",
      "epoth: 3, iter_num: 610, loss: 1.6749, 24.40%\n",
      "epoth: 3, iter_num: 620, loss: 1.1642, 24.80%\n",
      "epoth: 3, iter_num: 630, loss: 0.8131, 25.20%\n",
      "epoth: 3, iter_num: 640, loss: 1.3968, 25.60%\n",
      "epoth: 3, iter_num: 650, loss: 1.5615, 26.00%\n",
      "epoth: 3, iter_num: 660, loss: 1.8712, 26.40%\n",
      "epoth: 3, iter_num: 670, loss: 1.1584, 26.80%\n",
      "epoth: 3, iter_num: 680, loss: 1.8700, 27.20%\n",
      "epoth: 3, iter_num: 690, loss: 1.9061, 27.60%\n",
      "epoth: 3, iter_num: 700, loss: 1.4864, 28.00%\n",
      "epoth: 3, iter_num: 710, loss: 1.9469, 28.40%\n",
      "epoth: 3, iter_num: 720, loss: 1.4378, 28.80%\n",
      "epoth: 3, iter_num: 730, loss: 1.6604, 29.20%\n",
      "epoth: 3, iter_num: 740, loss: 1.8446, 29.60%\n",
      "epoth: 3, iter_num: 750, loss: 1.5248, 30.00%\n",
      "epoth: 3, iter_num: 760, loss: 1.6409, 30.40%\n",
      "epoth: 3, iter_num: 770, loss: 1.2522, 30.80%\n",
      "epoth: 3, iter_num: 780, loss: 1.2959, 31.20%\n",
      "epoth: 3, iter_num: 790, loss: 1.4919, 31.60%\n",
      "epoth: 3, iter_num: 800, loss: 1.5775, 32.00%\n",
      "epoth: 3, iter_num: 810, loss: 1.3496, 32.40%\n",
      "epoth: 3, iter_num: 820, loss: 1.8721, 32.80%\n",
      "epoth: 3, iter_num: 830, loss: 1.6910, 33.20%\n",
      "epoth: 3, iter_num: 840, loss: 1.3710, 33.60%\n",
      "epoth: 3, iter_num: 850, loss: 1.3712, 34.00%\n",
      "epoth: 3, iter_num: 860, loss: 1.4498, 34.40%\n",
      "epoth: 3, iter_num: 870, loss: 1.2812, 34.80%\n",
      "epoth: 3, iter_num: 880, loss: 1.3634, 35.20%\n",
      "epoth: 3, iter_num: 890, loss: 1.6219, 35.60%\n",
      "epoth: 3, iter_num: 900, loss: 1.5164, 36.00%\n",
      "epoth: 3, iter_num: 910, loss: 1.6297, 36.40%\n",
      "epoth: 3, iter_num: 920, loss: 1.7939, 36.80%\n",
      "epoth: 3, iter_num: 930, loss: 1.1644, 37.20%\n",
      "epoth: 3, iter_num: 940, loss: 1.7981, 37.60%\n",
      "epoth: 3, iter_num: 950, loss: 1.1659, 38.00%\n",
      "epoth: 3, iter_num: 960, loss: 0.9583, 38.40%\n",
      "epoth: 3, iter_num: 970, loss: 1.6180, 38.80%\n",
      "epoth: 3, iter_num: 980, loss: 1.1562, 39.20%\n",
      "epoth: 3, iter_num: 990, loss: 1.4298, 39.60%\n",
      "epoth: 3, iter_num: 1000, loss: 1.2248, 40.00%\n",
      "epoth: 3, iter_num: 1010, loss: 1.3669, 40.40%\n",
      "epoth: 3, iter_num: 1020, loss: 1.5433, 40.80%\n",
      "epoth: 3, iter_num: 1030, loss: 1.2552, 41.20%\n",
      "epoth: 3, iter_num: 1040, loss: 1.1951, 41.60%\n",
      "epoth: 3, iter_num: 1050, loss: 1.2239, 42.00%\n",
      "epoth: 3, iter_num: 1060, loss: 1.3817, 42.40%\n",
      "epoth: 3, iter_num: 1070, loss: 1.7163, 42.80%\n",
      "epoth: 3, iter_num: 1080, loss: 1.4247, 43.20%\n",
      "epoth: 3, iter_num: 1090, loss: 1.6127, 43.60%\n",
      "epoth: 3, iter_num: 1100, loss: 2.0508, 44.00%\n",
      "epoth: 3, iter_num: 1110, loss: 1.5418, 44.40%\n",
      "epoth: 3, iter_num: 1120, loss: 1.4243, 44.80%\n",
      "epoth: 3, iter_num: 1130, loss: 1.3265, 45.20%\n",
      "epoth: 3, iter_num: 1140, loss: 1.6681, 45.60%\n",
      "epoth: 3, iter_num: 1150, loss: 1.6375, 46.00%\n",
      "epoth: 3, iter_num: 1160, loss: 1.6423, 46.40%\n",
      "epoth: 3, iter_num: 1170, loss: 1.4133, 46.80%\n",
      "epoth: 3, iter_num: 1180, loss: 1.0750, 47.20%\n",
      "epoth: 3, iter_num: 1190, loss: 1.4224, 47.60%\n",
      "epoth: 3, iter_num: 1200, loss: 1.5240, 48.00%\n",
      "epoth: 3, iter_num: 1210, loss: 1.4832, 48.40%\n",
      "epoth: 3, iter_num: 1220, loss: 1.6926, 48.80%\n",
      "epoth: 3, iter_num: 1230, loss: 1.3896, 49.20%\n",
      "epoth: 3, iter_num: 1240, loss: 1.3657, 49.60%\n",
      "epoth: 3, iter_num: 1250, loss: 1.6092, 50.00%\n",
      "epoth: 3, iter_num: 1260, loss: 1.3943, 50.40%\n",
      "epoth: 3, iter_num: 1270, loss: 1.5327, 50.80%\n",
      "epoth: 3, iter_num: 1280, loss: 1.7966, 51.20%\n",
      "epoth: 3, iter_num: 1290, loss: 1.5743, 51.60%\n",
      "epoth: 3, iter_num: 1300, loss: 1.2088, 52.00%\n",
      "epoth: 3, iter_num: 1310, loss: 2.0825, 52.40%\n",
      "epoth: 3, iter_num: 1320, loss: 1.3183, 52.80%\n",
      "epoth: 3, iter_num: 1330, loss: 1.5195, 53.20%\n",
      "epoth: 3, iter_num: 1340, loss: 1.5425, 53.60%\n",
      "epoth: 3, iter_num: 1350, loss: 1.4929, 54.00%\n",
      "epoth: 3, iter_num: 1360, loss: 1.1643, 54.40%\n",
      "epoth: 3, iter_num: 1370, loss: 1.9535, 54.80%\n",
      "epoth: 3, iter_num: 1380, loss: 1.8784, 55.20%\n",
      "epoth: 3, iter_num: 1390, loss: 1.6576, 55.60%\n",
      "epoth: 3, iter_num: 1400, loss: 1.1247, 56.00%\n",
      "epoth: 3, iter_num: 1410, loss: 1.4193, 56.40%\n",
      "epoth: 3, iter_num: 1420, loss: 1.4758, 56.80%\n",
      "epoth: 3, iter_num: 1430, loss: 1.2598, 57.20%\n",
      "epoth: 3, iter_num: 1440, loss: 1.4289, 57.60%\n",
      "epoth: 3, iter_num: 1450, loss: 1.8887, 58.00%\n",
      "epoth: 3, iter_num: 1460, loss: 1.2689, 58.40%\n",
      "epoth: 3, iter_num: 1470, loss: 1.5645, 58.80%\n",
      "epoth: 3, iter_num: 1480, loss: 1.6232, 59.20%\n",
      "epoth: 3, iter_num: 1490, loss: 1.6165, 59.60%\n",
      "epoth: 3, iter_num: 1500, loss: 1.6683, 60.00%\n",
      "epoth: 3, iter_num: 1510, loss: 1.0643, 60.40%\n",
      "epoth: 3, iter_num: 1520, loss: 1.1003, 60.80%\n",
      "epoth: 3, iter_num: 1530, loss: 1.4420, 61.20%\n",
      "epoth: 3, iter_num: 1540, loss: 1.3078, 61.60%\n",
      "epoth: 3, iter_num: 1550, loss: 1.3713, 62.00%\n",
      "epoth: 3, iter_num: 1560, loss: 1.5535, 62.40%\n",
      "epoth: 3, iter_num: 1570, loss: 1.8438, 62.80%\n",
      "epoth: 3, iter_num: 1580, loss: 1.2343, 63.20%\n",
      "epoth: 3, iter_num: 1590, loss: 1.4199, 63.60%\n",
      "epoth: 3, iter_num: 1600, loss: 1.6320, 64.00%\n",
      "epoth: 3, iter_num: 1610, loss: 1.9283, 64.40%\n",
      "epoth: 3, iter_num: 1620, loss: 1.1591, 64.80%\n",
      "epoth: 3, iter_num: 1630, loss: 1.2333, 65.20%\n",
      "epoth: 3, iter_num: 1640, loss: 1.1464, 65.60%\n",
      "epoth: 3, iter_num: 1650, loss: 1.4716, 66.00%\n",
      "epoth: 3, iter_num: 1660, loss: 1.5360, 66.40%\n",
      "epoth: 3, iter_num: 1670, loss: 2.0306, 66.80%\n",
      "epoth: 3, iter_num: 1680, loss: 1.5393, 67.20%\n",
      "epoth: 3, iter_num: 1690, loss: 1.5994, 67.60%\n",
      "epoth: 3, iter_num: 1700, loss: 1.4703, 68.00%\n",
      "epoth: 3, iter_num: 1710, loss: 1.1315, 68.40%\n",
      "epoth: 3, iter_num: 1720, loss: 1.9008, 68.80%\n",
      "epoth: 3, iter_num: 1730, loss: 1.7133, 69.20%\n",
      "epoth: 3, iter_num: 1740, loss: 1.2362, 69.60%\n",
      "epoth: 3, iter_num: 1750, loss: 1.4607, 70.00%\n",
      "epoth: 3, iter_num: 1760, loss: 1.4498, 70.40%\n",
      "epoth: 3, iter_num: 1770, loss: 1.4435, 70.80%\n",
      "epoth: 3, iter_num: 1780, loss: 1.7799, 71.20%\n",
      "epoth: 3, iter_num: 1790, loss: 1.7356, 71.60%\n",
      "epoth: 3, iter_num: 1800, loss: 1.2809, 72.00%\n",
      "epoth: 3, iter_num: 1810, loss: 1.3398, 72.40%\n",
      "epoth: 3, iter_num: 1820, loss: 1.1661, 72.80%\n",
      "epoth: 3, iter_num: 1830, loss: 1.6786, 73.20%\n",
      "epoth: 3, iter_num: 1840, loss: 1.9771, 73.60%\n",
      "epoth: 3, iter_num: 1850, loss: 1.7728, 74.00%\n",
      "epoth: 3, iter_num: 1860, loss: 1.3612, 74.40%\n",
      "epoth: 3, iter_num: 1870, loss: 1.4854, 74.80%\n",
      "epoth: 3, iter_num: 1880, loss: 1.4068, 75.20%\n",
      "epoth: 3, iter_num: 1890, loss: 1.8930, 75.60%\n",
      "epoth: 3, iter_num: 1900, loss: 1.4159, 76.00%\n",
      "epoth: 3, iter_num: 1910, loss: 1.6427, 76.40%\n",
      "epoth: 3, iter_num: 1920, loss: 1.8276, 76.80%\n",
      "epoth: 3, iter_num: 1930, loss: 1.6154, 77.20%\n",
      "epoth: 3, iter_num: 1940, loss: 1.6666, 77.60%\n",
      "epoth: 3, iter_num: 1950, loss: 1.4535, 78.00%\n",
      "epoth: 3, iter_num: 1960, loss: 1.6031, 78.40%\n",
      "epoth: 3, iter_num: 1970, loss: 1.4591, 78.80%\n",
      "epoth: 3, iter_num: 1980, loss: 1.8127, 79.20%\n",
      "epoth: 3, iter_num: 1990, loss: 1.7399, 79.60%\n",
      "epoth: 3, iter_num: 2000, loss: 1.2044, 80.00%\n",
      "epoth: 3, iter_num: 2010, loss: 1.3663, 80.40%\n",
      "epoth: 3, iter_num: 2020, loss: 1.3602, 80.80%\n",
      "epoth: 3, iter_num: 2030, loss: 1.6600, 81.20%\n",
      "epoth: 3, iter_num: 2040, loss: 1.5646, 81.60%\n",
      "epoth: 3, iter_num: 2050, loss: 1.7623, 82.00%\n",
      "epoth: 3, iter_num: 2060, loss: 1.6545, 82.40%\n",
      "epoth: 3, iter_num: 2070, loss: 1.6615, 82.80%\n",
      "epoth: 3, iter_num: 2080, loss: 1.6814, 83.20%\n",
      "epoth: 3, iter_num: 2090, loss: 1.3122, 83.60%\n",
      "epoth: 3, iter_num: 2100, loss: 1.7457, 84.00%\n",
      "epoth: 3, iter_num: 2110, loss: 1.6433, 84.40%\n",
      "epoth: 3, iter_num: 2120, loss: 1.5396, 84.80%\n",
      "epoth: 3, iter_num: 2130, loss: 1.4652, 85.20%\n",
      "epoth: 3, iter_num: 2140, loss: 1.3444, 85.60%\n",
      "epoth: 3, iter_num: 2150, loss: 1.7768, 86.00%\n",
      "epoth: 3, iter_num: 2160, loss: 1.3177, 86.40%\n",
      "epoth: 3, iter_num: 2170, loss: 0.9612, 86.80%\n",
      "epoth: 3, iter_num: 2180, loss: 1.6188, 87.20%\n",
      "epoth: 3, iter_num: 2190, loss: 1.1451, 87.60%\n",
      "epoth: 3, iter_num: 2200, loss: 1.6117, 88.00%\n",
      "epoth: 3, iter_num: 2210, loss: 1.5137, 88.40%\n",
      "epoth: 3, iter_num: 2220, loss: 1.7351, 88.80%\n",
      "epoth: 3, iter_num: 2230, loss: 1.2955, 89.20%\n",
      "epoth: 3, iter_num: 2240, loss: 1.3156, 89.60%\n",
      "epoth: 3, iter_num: 2250, loss: 1.0328, 90.00%\n",
      "epoth: 3, iter_num: 2260, loss: 1.6893, 90.40%\n",
      "epoth: 3, iter_num: 2270, loss: 1.3079, 90.80%\n",
      "epoth: 3, iter_num: 2280, loss: 1.8105, 91.20%\n",
      "epoth: 3, iter_num: 2290, loss: 1.6326, 91.60%\n",
      "epoth: 3, iter_num: 2300, loss: 1.2445, 92.00%\n",
      "epoth: 3, iter_num: 2310, loss: 1.5403, 92.40%\n",
      "epoth: 3, iter_num: 2320, loss: 1.5367, 92.80%\n",
      "epoth: 3, iter_num: 2330, loss: 1.7816, 93.20%\n",
      "epoth: 3, iter_num: 2340, loss: 1.6623, 93.60%\n",
      "epoth: 3, iter_num: 2350, loss: 1.6501, 94.00%\n",
      "epoth: 3, iter_num: 2360, loss: 1.7559, 94.40%\n",
      "epoth: 3, iter_num: 2370, loss: 1.3506, 94.80%\n",
      "epoth: 3, iter_num: 2380, loss: 1.6500, 95.20%\n",
      "epoth: 3, iter_num: 2390, loss: 1.2946, 95.60%\n",
      "epoth: 3, iter_num: 2400, loss: 1.3987, 96.00%\n",
      "epoth: 3, iter_num: 2410, loss: 1.6454, 96.40%\n",
      "epoth: 3, iter_num: 2420, loss: 1.5717, 96.80%\n",
      "epoth: 3, iter_num: 2430, loss: 1.4207, 97.20%\n",
      "epoth: 3, iter_num: 2440, loss: 1.2099, 97.60%\n",
      "epoth: 3, iter_num: 2450, loss: 1.3352, 98.00%\n",
      "epoth: 3, iter_num: 2460, loss: 1.6951, 98.40%\n",
      "epoth: 3, iter_num: 2470, loss: 1.5517, 98.80%\n",
      "epoth: 3, iter_num: 2480, loss: 1.3793, 99.20%\n",
      "epoth: 3, iter_num: 2490, loss: 1.6802, 99.60%\n",
      "epoth: 3, iter_num: 2500, loss: 1.7091, 100.00%\n",
      "Epoch: 3, Average training loss: 1.5026\n",
      "Accuracy: 0.6164\n",
      "Average testing loss: 1.4032\n",
      "-------------------------------\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "# 训练函数\n",
    "def train():\n",
    "    model.train()\n",
    "    fgm = FGM(model)\n",
    "    total_train_loss = 0\n",
    "    iter_num = 0\n",
    "    total_iter = len(train_loader)\n",
    "    for batch in train_loader:\n",
    "        # 正向传播\n",
    "        optim.zero_grad()\n",
    "        \n",
    "        input_ids = batch['input_ids'].to(device)\n",
    "        attention_mask = batch['attention_mask'].to(device)\n",
    "        labels = batch['labels'].to(device)\n",
    "        \n",
    "        outputs = model(input_ids, attention_mask=attention_mask, labels=labels)\n",
    "        loss = outputs[0]\n",
    "        total_train_loss += loss.item()\n",
    "        \n",
    "        # 反向梯度信息\n",
    "        loss.backward()\n",
    "        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n",
    "        \n",
    "        # 产生对抗样本\n",
    "        # 再次正向传播\n",
    "         \n",
    "        # 对抗训练\n",
    "        fgm.attack() # embedding被修改了\n",
    "        loss_sum = model(input_ids, attention_mask=attention_mask, labels=labels)\n",
    "        loss_sum[0].backward() # 反向传播，在正常的grad基础上，累加对抗训练的梯度\n",
    "        fgm.restore() # 恢复Embedding的参数\n",
    "        \n",
    "        # 参数更新\n",
    "        optim.step()\n",
    "        scheduler.step()\n",
    "\n",
    "        iter_num += 1\n",
    "        if(iter_num % 10==0):\n",
    "            print(\"epoth: %d, iter_num: %d, loss: %.4f, %.2f%%\" % (epoch, iter_num, loss.item(), iter_num/total_iter*100))\n",
    "        \n",
    "    print(\"Epoch: %d, Average training loss: %.4f\"%(epoch, total_train_loss/len(train_loader)))\n",
    "    \n",
    "def validation():\n",
    "    model.eval()\n",
    "    total_eval_accuracy = 0\n",
    "    total_eval_loss = 0\n",
    "    for batch in test_dataloader:\n",
    "        with torch.no_grad():\n",
    "            # 正常传播\n",
    "            input_ids = batch['input_ids'].to(device)\n",
    "            attention_mask = batch['attention_mask'].to(device)\n",
    "            labels = batch['labels'].to(device)\n",
    "            outputs = model(input_ids, attention_mask=attention_mask, labels=labels)\n",
    "        \n",
    "        loss = outputs[0]\n",
    "        logits = outputs[1]\n",
    "\n",
    "        total_eval_loss += loss.item()\n",
    "        logits = logits.detach().cpu().numpy()\n",
    "        label_ids = labels.to('cpu').numpy()\n",
    "        total_eval_accuracy += flat_accuracy(logits, label_ids)\n",
    "        \n",
    "    avg_val_accuracy = total_eval_accuracy / len(test_dataloader)\n",
    "    print(\"Accuracy: %.4f\" % (avg_val_accuracy))\n",
    "    print(\"Average testing loss: %.4f\"%(total_eval_loss/len(test_dataloader)))\n",
    "    print(\"-------------------------------\")\n",
    "    \n",
    "\n",
    "for epoch in range(4):\n",
    "    print(\"------------Epoch: %d ----------------\" % epoch)\n",
    "    train()\n",
    "    validation()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
